Merge remote-tracking branch 'origin/GT-3341_emteere_RTTIPerformance'

Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java

@@ -15,23 +15,27 @@
  */
 package ghidra.app.plugin.core.analysis;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
 
 import ghidra.app.cmd.data.CreateDataCmd;
 import ghidra.app.services.*;
 import ghidra.app.util.importer.MessageLog;
 import ghidra.framework.options.Options;
-import ghidra.program.model.address.*;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.address.AddressSetView;
 import ghidra.program.model.data.*;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.BookmarkType;
+import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.Memory;
+import ghidra.util.bytesearch.*;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
 public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
 	private static final String NAME = "Embedded Media";
 	private static final String DESCRIPTION =
-		"Finds and tries to apply embedded media data types (ie png, gif, jpeg, wav) in current program.";
+		"Finds embedded media data types (ie png, gif, jpeg, wav)";
 
 	private static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks";
 	private static final String OPTION_DESCRIPTION_CREATE_BOOKMARKS =
@@ -52,92 +56,78 @@ public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
 			throws CancelledException {
 
 		Memory memory = program.getMemory();
-		AddressSetView initializedAddressSet = memory.getLoadedAndInitializedAddressSet();
+		AddressSetView validMemorySet = memory.getLoadedAndInitializedAddressSet();
-		AddressSet initialedSearchSet = set.intersect(initializedAddressSet);
+		AddressSetView searchSet = set.intersect(validMemorySet);
+		if (searchSet.isEmpty()) {
+			return false;  // no valid addresses to search
+		}
+
+		MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("Embedded Media");
 
 		List<Address> foundMedia = new ArrayList<>();
 
-		foundMedia = scanForMedia(program, new GifDataType(), "GIF 87", GifDataType.MAGIC_87,
+		addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 87",
-			GifDataType.GIFMASK, initialedSearchSet, memory, monitor);
+			GifDataType.MAGIC_87, GifDataType.GIFMASK);
 
-		foundMedia.addAll(scanForMedia(program, new GifDataType(), "GIF 89", GifDataType.MAGIC_89,
+		addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 89",
-			GifDataType.GIFMASK, initialedSearchSet, memory, monitor));
+			GifDataType.MAGIC_89, GifDataType.GIFMASK);
 
-		foundMedia.addAll(scanForMedia(program, new PngDataType(), "PNG", PngDataType.MAGIC,
+		addByteSearchPattern(searcher, program, foundMedia, new PngDataType(), "PNG",
-			PngDataType.MASK, initialedSearchSet, memory, monitor));
+			PngDataType.MAGIC, PngDataType.MASK);
 
-		foundMedia.addAll(scanForMedia(program, new JPEGDataType(), "JPEG", JPEGDataType.MAGIC,
+		addByteSearchPattern(searcher, program, foundMedia, new JPEGDataType(), "JPEG",
-			JPEGDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
+			JPEGDataType.MAGIC, JPEGDataType.MAGIC_MASK);
 
-		foundMedia.addAll(scanForMedia(program, new WAVEDataType(), "WAVE", WAVEDataType.MAGIC,
+		addByteSearchPattern(searcher, program, foundMedia, new WAVEDataType(), "WAVE",
-			WAVEDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
+			WAVEDataType.MAGIC, WAVEDataType.MAGIC_MASK);
 
-		foundMedia.addAll(scanForMedia(program, new AUDataType(), "AU", AUDataType.MAGIC,
+		addByteSearchPattern(searcher, program, foundMedia, new AUDataType(), "AU",
-			AUDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
+			AUDataType.MAGIC, AUDataType.MAGIC_MASK);
 
-		foundMedia.addAll(scanForMedia(program, new AIFFDataType(), "AIFF", AIFFDataType.MAGIC,
+		addByteSearchPattern(searcher, program, foundMedia, new AIFFDataType(), "AIFF",
-			AIFFDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
+			AIFFDataType.MAGIC, AIFFDataType.MAGIC_MASK);
 
-		return true;
+		searcher.search(program, searchSet, monitor);
+
+		return foundMedia.size() > 0;
 	}
 
-	private List<Address> scanForMedia(Program program, DataType dt, String mediaName,
+	private void addByteSearchPattern(MemoryBytePatternSearcher searcher, Program program,
-			byte[] mediaBytes, byte[] mask, AddressSetView addresses, Memory memory,
+			List<Address> foundMedia, DataType mediaDT, String mediaName, byte[] bytes,
-			TaskMonitor monitor) {
+			byte[] mask) {
-
+		if (bytes == null) {
-		monitor.setMessage("Scanning for " + mediaName + " Embedded Media");
+			return;
-		monitor.initialize(addresses.getNumAddresses());
-
-		List<Address> foundMediaAddresses = new ArrayList<>();
-
-		Iterator<AddressRange> iterator = addresses.iterator();
-		while (iterator.hasNext()) {
-			if (monitor.isCancelled()) {
-				return foundMediaAddresses;
-			}
-
-			AddressRange range = iterator.next();
-			Address start = range.getMinAddress();
-			Address end = range.getMaxAddress();
-
-			Address found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
-			while (found != null && !monitor.isCancelled()) {
-				//See if it is already an applied media data type
-				Data data = program.getListing().getDefinedDataAt(found);
-				int skipLen = 1;
-				if (data == null) {
-					try {
-						CreateDataCmd cmd = new CreateDataCmd(found, dt);
-						if (cmd.applyTo(program)) {
-							if (createBookmarksEnabled) {
-								program.getBookmarkManager().setBookmark(found,
-									BookmarkType.ANALYSIS, "Embedded Media",
-									"Found " + mediaName + " Embedded Media");
-							}
-							foundMediaAddresses.add(found);
-							//have to get the actual applied data to find the actual length to skip because until then it can't compute the length due to the data type being dynamic
-							skipLen = program.getListing().getDataAt(found).getLength();
-						}
-					}
-					//If media does not apply correctly then it is not really a that media data type or there is other data in the way
-					catch (Exception e) {
-						// Not a valid embedded media or no room to apply it so just ignore it and skip it
-					}
-				}
-				// skip either the valid data that was found or skip one byte
-				// then do the next search
-				try {
-					start = found.add(skipLen);
-					found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
-				}
-				catch (AddressOutOfBoundsException e) {
-					// If media was at the very end of the address space, we will end up here
-					break;
-				}
-			}
 		}
 
-		return foundMediaAddresses;
+		GenericMatchAction<DataType> action = new GenericMatchAction<DataType>(mediaDT) {
+			@Override
+			public void apply(Program prog, Address addr, Match match) {
+				//See if it is already an applied media data type
+				if (!program.getListing().isUndefined(addr, addr)) {
+					return;
+				}
+
+				try {
+					CreateDataCmd cmd = new CreateDataCmd(addr, mediaDT);
+					if (cmd.applyTo(program)) {
+						if (createBookmarksEnabled) {
+							program.getBookmarkManager().setBookmark(addr, BookmarkType.ANALYSIS,
+								"Embedded Media", "Found " + mediaName + " Embedded Media");
+						}
+						foundMedia.add(addr);
+					}
+				}
+				//If media does not apply correctly then it is not really a that media data type or there is other data in the way
+				catch (Exception e) {
+					// Not a valid embedded media or no room to apply it so just ignore it and skip it
+				}
+			}
+		};
+
+		GenericByteSequencePattern<DataType> genericByteMatchPattern =
+			new GenericByteSequencePattern<DataType>(bytes, mask, action);
+
+		searcher.addPattern(genericByteMatchPattern);
 	}
 
 	@Override

Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java

@@ -177,8 +177,7 @@ public class ProgramMemoryUtil {
 		MemoryBlock[] tmpBlocks = new MemoryBlock[blocks.length];
 		int j = 0;
 		for (MemoryBlock block : blocks) {
-			if ((block.isInitialized() && withBytes) ||
+			if ((block.isInitialized() && withBytes) || (!block.isInitialized() && !withBytes)) {
-				(!block.isInitialized() && !withBytes)) {
 				tmpBlocks[j++] = block;
 			}
 		}
@@ -493,6 +492,30 @@ public class ProgramMemoryUtil {
 			monitor = TaskMonitorAdapter.DUMMY_MONITOR;
 		}
 
+		byte[] addressBytes = getDirectAddressBytes(program, toAddress);
+
+		byte[] shiftedAddressBytes = getShiftedDirectAddressBytes(program, toAddress);
+
+		Memory memory = program.getMemory();
+		Set<Address> dirRefsAddrs = new TreeSet<>();
+		findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
+
+		if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
+			findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
+		}
+
+		return dirRefsAddrs;
+	}
+
+	/**
+	 * Get a representation of an address as it would appear in bytes in memory.
+	 * 
+	 * @param program program
+	 * @param toAddress target address
+	 * @return byte representation of toAddress
+	 */
+	public static byte[] getDirectAddressBytes(Program program, Address toAddress) {
+
 		Memory memory = program.getMemory();
 		boolean isBigEndian = memory.isBigEndian();
 
@@ -536,6 +559,31 @@ public class ProgramMemoryUtil {
 				addressBytes, 0, addressBytes.length);
 		}
 
+		return addressBytes;
+	}
+
+	/**
+	 * returns shifted address bytes if they are different than un-shifted
+	 * 
+	 * @param program program
+	 * @param toAddress target address
+	 * @return shifted bytes, null if same as un-shifted
+	 */
+	public static byte[] getShiftedDirectAddressBytes(Program program, Address toAddress) {
+
+		byte[] addressBytes = getDirectAddressBytes(program, toAddress);
+
+		Memory memory = program.getMemory();
+		boolean isBigEndian = memory.isBigEndian();
+
+		DataConverter dataConverter;
+		if (isBigEndian) {
+			dataConverter = new BigEndianDataConverter();
+		}
+		else {
+			dataConverter = new LittleEndianDataConverter();
+		}
+
 		byte[] shiftedAddressBytes = null;
 		DataTypeManager dataTypeManager = program.getDataTypeManager();
 		DataOrganization dataOrganization = dataTypeManager.getDataOrganization();
@@ -554,24 +602,23 @@ public class ProgramMemoryUtil {
 			}
 		}
 
-		// don't need this anymore - finding all 16 bit addrs in whole prog
+		return shiftedAddressBytes;
-//				AddressRange segmentRange = null;
+	}
-//				if (toAddress instanceof SegmentedAddress) {
-//					// Restrict search to currentSegment range
-//					SegmentedAddressSpace segSpace = (SegmentedAddressSpace) toAddress.getAddressSpace();
-//					segmentRange =
-//						new AddressRangeImpl(segSpace.getAddress(currentSegment, 0), segSpace.getAddress(
-//							currentSegment, 0xffff));
-//				}
 
-		Set<Address> dirRefsAddrs = new TreeSet<>();
+	public static byte[] getImageBaseOffsets32Bytes(Program program, int alignment,
-		findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
+			Address toAddress) {
 
-		if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
+		Address imageBase = program.getImageBase();
-			findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
+
+		long offsetValue = toAddress.subtract(imageBase);
+		int offsetSize = 4; // 32 bit offset
+		byte[] bytes = new byte[offsetSize];
+		for (int i = 0; i < offsetSize; i++) {
+			bytes[i] = (byte) offsetValue;
+			offsetValue >>= 8; // Shift by a single byte.
 		}
 
-		return dirRefsAddrs;
+		return bytes;
 	}
 
 	/**
@@ -627,8 +674,7 @@ public class ProgramMemoryUtil {
 			if (!block.isInitialized()) {
 				continue;
 			}
-			if (memoryRange != null &&
+			if (memoryRange != null && !memoryRange.intersects(block.getStart(), block.getEnd())) {
-				!memoryRange.intersects(block.getStart(), block.getEnd())) {
 				// skip blocks which do not correspond to currentSeg
 				continue;
 			}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java

@@ -0,0 +1,82 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.util.bytesearch;
+
+import ghidra.util.xml.SpecXmlUtils;
+import ghidra.xml.XmlElement;
+import ghidra.xml.XmlPullParser;
+
+/**
+ * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
+ * alignment at an offset from the location the pattern matches. 
+ * 
+ * The pattern can be constructed or restored from XML of the form,
+ * where alignOffset=mark, alignmask=bits
+ * 
+ *     <align mark="0" bits="1"/>
+ *   
+ */
+
+public class AlignRule implements PostRule {
+
+	private int alignOffset;  // Position, relative to start of pattern, to check alignment at
+	private int alignmask;    // Mask of bits that must be zero
+
+	public AlignRule() {
+	}
+
+	/**
+	 * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
+	 * alignment at an offset from the location the pattern matches. The alignment is
+	 * specified by the alignmask bits that must be zero.
+	 * 
+	 *   Normally alignOffset is 0, since most patterns will match at the address that must be aligned
+	 *   To align a match, use the following
+	 *
+	 *  align to  2 = alignmask 0x1 - lower bit must be zero
+	 *  align to  4 = alignmask 0x3 - lower two bits must be zero
+	 *  align to  8 = alignmask 0x7 - lower three bits must be zero
+	 *  align to 16 = alignmask 0xF - lower four bits must be zero
+	 *  ....
+	 *  Other strange alignments could be specified, but most likely the above suffice.
+	 * @param alignOffset - bytes offset from pattern to check for alignment
+	 * @param alignmask - the mask where a 1 bit must be zero
+	 */
+	public AlignRule(int alignOffset, int alignmask) {
+		this.alignOffset = alignOffset;
+		this.alignmask = alignmask;
+	}
+
+	@Override
+	public boolean apply(Pattern pat, long matchoffset) {
+		int off = (int) matchoffset;
+		return (((off + alignOffset) & alignmask) == 0);
+	}
+
+	@Override
+	public void restoreXml(XmlPullParser parser) {
+		XmlElement el = parser.start("align");
+		alignOffset = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
+		int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
+		alignmask = (1 << bits) - 1;
+		parser.end();
+	}
+
+	public int getAlignMask() {
+		return alignmask;
+	}
+
+}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java

@@ -21,6 +21,19 @@ import java.util.zip.CRC32;
 
 import ghidra.xml.XmlPullParser;
 
+/**
+ * A pattern of bits/mask to match to a stream of bytes.  The bits/mask can be of any length.
+ * The sequence can be initialized by:
+ * 
+ *    a string
+ *    an array of bytes (no mask)
+ *    an array of bytes and for mask
+ *    
+ *  The dits represent bits(binary) or nibbles(hex) that are don't care, for example:
+ *     0x..d.4de2 ....0000 .1...... 00101101 11101001
+ *  where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
+ */
+
 public class DittedBitSequence {
 
 	//Given a byte 0-255 (NOT a signed byte), retrieves its popcount.
@@ -42,7 +55,7 @@ public class DittedBitSequence {
 		4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8  //240-255
 	};
 
-	protected int index;		// Unique index assigned to this sequence
+	private int index;		// Unique index assigned to this sequence
 	private byte[] bits;		// value bits contained in the sequence
 	private byte[] dits;		// a 1 indicates the bit is not ditted
 
@@ -53,7 +66,9 @@ public class DittedBitSequence {
 
 	/**
 	 * Constructor from a ditted-bit-sequence string where white space is ignored (e.g., "10..11.0");
-	 * @param dittedBitData
+	 * 
+	 * @param dittedBitData ditted sequence specified as a string
+	 * 
 	 * @throws IllegalArgumentException if invalid dittedBitData specified
 	 */
 	public DittedBitSequence(String dittedBitData) {
@@ -64,8 +79,8 @@ public class DittedBitSequence {
 	 * Constructor from a ditted-bit string where white space is ignored.  If there are no dits,
 	 * {@code hex} is true, and {@code hex} does not begin with {code 0x}, {@code 0x} will be
 	 * prepended to the string before constructing the {@link DittedBitSequence}.
-	 * @param dittedBitData
+	 * @param dittedBitData string of bits and dits or hex numbers and dits (e.g., 0.1..0, 0xAB..)
-	 * @param hex
+	 * @param hex true to force hex on the sequence
 	 */
 	public DittedBitSequence(String dittedBitData, boolean hex) {
 		if (hex && !dittedBitData.contains(".")) {
@@ -101,7 +116,8 @@ public class DittedBitSequence {
 
 	/**
 	 * Construct a sequence of bytes to search for. No bits are masked off.
-	 * @param bytes
+	 * 
+	 * @param bytes byte values that must match
 	 */
 	public DittedBitSequence(byte[] bytes) {
 		bits = bytes;
@@ -164,22 +180,40 @@ public class DittedBitSequence {
 		return true;
 	}
 
-	public DittedBitSequence concatenate(DittedBitSequence op2) {
+	/**
+	 * Concatenates a sequence to the end of another sequence and
+	 * returns a new sequence.
+	 * 
+	 * @param toConat sequence to concatenate to this sequence
+	 * 
+	 * @return a new sequence that is the concat of this and toConcat
+	 */
+	public DittedBitSequence concatenate(DittedBitSequence toConat) {
 		DittedBitSequence res = new DittedBitSequence();
-		res.bits = new byte[bits.length + op2.bits.length];
+		res.bits = new byte[bits.length + toConat.bits.length];
 		res.dits = new byte[res.bits.length];
 		for (int i = 0; i < bits.length; ++i) {
 			res.bits[i] = bits[i];
 			res.dits[i] = dits[i];
 		}
-		for (int i = 0; i < op2.bits.length; ++i) {
+		for (int i = 0; i < toConat.bits.length; ++i) {
-			res.bits[bits.length + i] = op2.bits[i];
+			res.bits[bits.length + i] = toConat.bits[i];
-			res.dits[bits.length + i] = op2.dits[i];
+			res.dits[bits.length + i] = toConat.dits[i];
 		}
 
 		return res;
 	}
 
+	/**
+	 * Check for a match of a value at a certain offset in the pattern.
+	 * An outside matcher will keep track of the match position within this
+	 * ditted bit sequence.  Then call this method to match.
+	 * 
+	 * @param pos position in the pattern to match
+	 * @param val a byte to be match at the given byte offset in the pattern
+	 * 
+	 * @return true if the byte matches the sequence mask/value
+	 */
 	public boolean isMatch(int pos, int val) {
 		if (pos >= bits.length) {
 			return false;
@@ -187,14 +221,38 @@ public class DittedBitSequence {
 		return ((byte) (val & dits[pos])) == bits[pos];
 	}
 
+	/**
+	 * Set a an index in a larger sequence, or identifing id on this pattern
+	 * 
+	 * @param index - index in match sequence, or unique id
+	 */
+	public void setIndex(int index) {
+		this.index = index;
+	}
+
+	/**
+	 * Get the index or identifying id attached to this pattern
+	 * 
+	 * @return index or unique id attached to this sequence
+	 */
 	public int getIndex() {
 		return index;
 	}
 
+	/**
+	 * get the size of this sequence in bytes
+	 * 
+	 * @return size in bytes
+	 */
 	public int getSize() {
 		return bits.length;
 	}
 
+	/**
+	 * Get number of bits that must be 0/1
+	 * 
+	 * @return number of bits that are not don't care (ditted)
+	 */
 	public int getNumFixedBits() {
 		int popcnt = 0;
 		for (byte dit : dits) {
@@ -203,7 +261,11 @@ public class DittedBitSequence {
 		return popcnt;
 	}
 
-	//Return the number of dits.
+	/**
+	 * Get number of bits that are ditted (don't care)
+	 * 
+	 * @return number of ditted bits (don't care)
+	 */
 	public int getNumUncertainBits() {
 		int popcnt = 0;
 		for (byte dit : dits) {
@@ -235,6 +297,11 @@ public class DittedBitSequence {
 		return buf.toString();
 	}
 
+	/**
+	 * get a ditted hex string representing this sequence
+	 * 
+	 * @return ditted hex string
+	 */
 	public String getHexString() {
 		String uncompressed = this.toString();
 		String[] parts = uncompressed.trim().split(" ");
@@ -260,6 +327,17 @@ public class DittedBitSequence {
 		return sb.toString();
 	}
 
+	/**
+	 * restore ditted string from XML stream with hex/binary ditted sequences in the form:
+	 *    <data> 0x..d.4de2 ....0000 .1...... 00101101 11101001 </data>
+	 * where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
+	 * 
+	 * @param parser XML pull parser stream
+	 * 
+	 * @return number of bytes read from XML <data> tag
+	 * 
+	 * @throws IOException if XML read has an error
+	 */
 	protected int restoreXmlData(XmlPullParser parser) throws IOException {
 		parser.start("data");
 		String text = parser.end().getText();
@@ -272,6 +350,16 @@ public class DittedBitSequence {
 		}
 	}
 
+	/**
+	 * Initialize this sequence with a ditted sequence from a string in the form
+	 *    (e.g. - 011...1., 0x.F, 01110011 0xAB)
+	 *    
+	 * @param text ditted sequence
+	 * 
+	 * @return number of bytes in the ditted sequence
+	 * 
+	 * @throws IllegalArgumentException if string is malformed
+	 */
 	private int initFromDittedStringData(String text) throws IllegalArgumentException {
 		int markOffset = -1;
 		int mode = -1; // -1: looking for start, -2: skip to EOL, 0: hex mode, 1: binary mode
@@ -372,6 +460,13 @@ public class DittedBitSequence {
 		return markOffset;
 	}
 
+	/**
+	 * Get the number of bits that are fixed, not ditted (don't care)
+	 * 
+	 * @param marked number of bytes in the pattern to check
+	 * 
+	 * @return number of initial fixed bits
+	 */
 	public int getNumInitialFixedBits(int marked) {
 		if (dits == null) {
 			return 0;
@@ -385,5 +480,4 @@ public class DittedBitSequence {
 		}
 		return popcnt;
 	}
-
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +19,9 @@ import ghidra.program.model.address.Address;
 import ghidra.program.model.listing.Program;
 import ghidra.xml.XmlPullParser;
 
+/**
+ * Dummy action attached to a match sequence.  Action is not restored from XML
+ */
 public class DummyMatchAction implements MatchAction {
 
 	@Override
@@ -30,6 +32,5 @@ public class DummyMatchAction implements MatchAction {
 	public void restoreXml(XmlPullParser parser) {
 		parser.discardSubTree();
 	}
-	
-}
 
+}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java

@@ -0,0 +1,61 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.util.bytesearch;
+
+import ghidra.program.model.data.DataType;
+
+/**
+ * Templated simple DittedBitSequence Pattern for a byte/mask pattern and associated action.
+ * The DittedBitSequence is provided by value and mask in byte arrays.
+ * 
+ * This class is normally used to find some number of SequencePatterns within a seqence of bytes.
+ * When the byte/mask pattern is matched, the GenericMatchAction will be "applied".
+ *
+ * @param <T> the class of match action, used to specify a specialized momento to be used by the action when it is "applied".
+ */
+
+public class GenericByteSequencePattern<T> extends Pattern {
+
+	/**
+	 * Construct a sequence of bytes with no mask, and associated action
+	 * to be called if this pattern matches.
+	 * 
+	 * @param bytesSequence sequence of bytes to match
+	 * @param action action to apply if the match succeeds
+	 */
+	public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction<T> action) {
+		super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]);
+
+		MatchAction[] matchActions = getMatchActions();
+		matchActions[0] = action;
+	}
+
+	/**
+	 * Construct a sequence of bytes with a mask, and associated action
+	 * to be called if this pattern matches.
+	 * 
+	 * @param bytesSequence sequence of bytes to match
+	 * @param mask mask, bits that are 1 must match the byteSequence bits
+	 * @param action to apply if the match succeeds
+	 */
+	public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask,
+			GenericMatchAction<DataType> action) {
+		super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]);
+
+		MatchAction[] matchActions = getMatchActions();
+		matchActions[0] = action;
+	}
+}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java

@@ -0,0 +1,43 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.util.bytesearch;
+
+/**
+ * Template for generic match action attached to a match sequence.
+ * Used to store an associated value to the matching sequence.
+ * The associated value can be retrieved when the sequence is matched.
+ *
+ * @param <T> - object to attach to match sequence, generally used to specify
+ * a specialized momento to be used by the action when it is "applied".
+ */
+public class GenericMatchAction<T> extends DummyMatchAction {
+	T matchValue;
+
+	/**
+	 * Construct a match action used when a match occurs for some GenericByteSequece
+	 * @param matchValue specialized object used when match occurs
+	 */
+	public GenericMatchAction(T matchValue) {
+		this.matchValue = matchValue;
+	}
+
+	/**
+	 * @return the specialized object associated with this match action
+	 */
+	public T getMatchValue() {
+		return this.matchValue;
+	}
+}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java

@@ -15,13 +15,27 @@
  */
 package ghidra.util.bytesearch;
 
+/**
+ * Represents a match of a DittedBitSequence at a given offset in a byte sequence.
+ * 
+ * There is a hidden assumption that the sequence is actually a Pattern
+ * that might have a ditted-bit-sequence, a set of match actions,
+ * and post match rules/checks
+ * 
+ */
 public class Match {
-	private DittedBitSequence sequence;	// Pattern that matches
+	private DittedBitSequence sequence;	// Pattern that matched
-	private long offset;			// starting offset within bytestream of match
+	private long offset;			    // Offset within bytestream where the match occurred
 
-	public Match(DittedBitSequence seq, long off) {
+	/**
-		sequence = seq;
+	 * Construct a Match of a DittedBitSequence at an offset within a byte stream.
-		offset = off;
+	 * Object normally used when a match occurs during a MemoryBytePatternSearch.
+	 * @param sequence that matched
+	 * @param offset from the start of byte stream where the matched occured
+	 */
+	public Match(DittedBitSequence sequence, long offset) {
+		this.sequence = sequence;
+		this.offset = offset;
 	}
 
 	/**
@@ -39,42 +53,70 @@ public class Match {
 		return sequence.getNumFixedBits() - sequence.getNumInitialFixedBits(marked);
 	}
 
+	/**
+	 * @return actions associated with this match
+	 */
 	public MatchAction[] getMatchActions() {
 		return ((Pattern) sequence).getMatchActions();
 	}
 
+	/**
+	 * @return size in bytes of sequence
+	 */
 	public int getSequenceSize() {
 		return sequence.getSize();
 	}
 
+	/**
+	 * @return index of sequence in a possibly longer set of sequences
+	 */
 	public int getSequenceIndex() {
 		return sequence.getIndex();
 	}
 
+	/**
+	 * @return the offset of the match within a longer byte sequence
+	 */
 	public long getMarkOffset() {
 		return offset + ((Pattern) sequence).getMarkOffset();
 	}
 
+	/** 
+	 * @return offset of match in sequence of bytes
+	 */
 	public long getMatchStart() {
 		return offset;
 	}
 
+	/**
+	 * Check that the possible post rules are satisfied
+	 * 
+	 * @param streamoffset offset within from match location to check postrules.
+	 * 
+	 * @return true if post rules are satisfied
+	 */
 	public boolean checkPostRules(long streamoffset) {
 		long curoffset = streamoffset + offset;
 		Pattern pattern = (Pattern) sequence;
 		PostRule[] postRules = pattern.getPostRules();
-		for (int i = 0; i < postRules.length; ++i) {
+		for (PostRule postRule : postRules) {
-			if (!postRules[i].apply(pattern, curoffset)) {
+			if (!postRule.apply(pattern, curoffset)) {
 				return false;
 			}
 		}
 		return true;
 	}
 
+	/**
+	 * @return ditted bit sequence as a string
+	 */
 	public String getHexString() {
 		return sequence.getHexString();
 	}
 
+	/**
+	 * @return the sequence that was matched
+	 */
 	public DittedBitSequence getSequence() {
 		return sequence;
 	}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +20,22 @@ import ghidra.program.model.listing.Program;
 import ghidra.xml.XmlPullParser;
 
 /**
- * Action that should be applied to a Program at the Address a pattern matches
+ * Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern
- *
  */
 public interface MatchAction {
-	public void apply(Program program,Address addr,Match match);
+	/**
-	
+	 * Apply the match action to the program at the address.
+	 * 
+	 * @param program program in which the match occurred
+	 * @param addr where the match occured
+	 * @param match information about the match that occurred
+	 */
+	public void apply(Program program, Address addr, Match match);
+
+	/**
+	 * Action can be constructed from XML
+	 * 
+	 * @param parser XML pull parser to restore action from XML
+	 */
 	public void restoreXml(XmlPullParser parser);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java

@@ -0,0 +1,268 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.util.bytesearch;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+
+import ghidra.program.model.address.*;
+import ghidra.program.model.listing.Program;
+import ghidra.program.model.mem.MemoryBlock;
+import ghidra.util.Msg;
+import ghidra.util.exception.CancelledException;
+import ghidra.util.task.TaskMonitor;
+
+/**
+ * Multi pattern/mask/action memory searcher
+ * Patterns must be supplied/added, or a pre-initialized searchState supplied
+ * 
+ * Preload search patterns and actions, then call search method.
+ */
+
+public class MemoryBytePatternSearcher {
+	private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;
+
+	SequenceSearchState root = null;
+
+	ArrayList<Pattern> patternList;
+
+	private String searchName = "";
+
+	private boolean doExecutableBlocksOnly = false;  // only search executable blocks
+
+	private long numToSearch = 0;
+	private long numSearched = 0;
+
+	/**
+	 * Create with pre-created patternList
+	 * @param searchName name of search
+	 * @param patternList - list of patterns(bytes/mask/action)
+	 */
+	public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {
+		this.searchName = searchName;
+		this.patternList = new ArrayList<Pattern>(patternList);
+	}
+
+	/**
+	 * Create with an initialized SequenceSearchState
+	 * @param searchName name of search
+	 * @param root search state pre-initialized
+	 */
+	public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) {
+		this.searchName = searchName;
+		this.root = root;
+	}
+
+	/**
+	 * Create with no patternList, must add patterns before searching
+	 * @param searchName name of search
+	 * 
+	 */
+	public MemoryBytePatternSearcher(String searchName) {
+		this.searchName = searchName;
+		patternList = new ArrayList<>();
+	}
+
+	/**
+	 * Add a search pattern 
+	 * @param pattern - pattern(bytes/mask/action)
+	 */
+	public void addPattern(Pattern pattern) {
+		patternList.add(pattern);
+	}
+
+	public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) {
+		this.doExecutableBlocksOnly = doExecutableBlocksOnly;
+	}
+
+	/**
+	 * Search initialized memory blocks for all patterns(bytes/mask/action).
+	 * Call associated action for each pattern matched.
+	 * 
+	 * @param program to be searched
+	 * @param searchSet set of bytes to restrict search, if null or empty then search all memory blocks
+	 * @param monitor allow canceling and reporting of progress
+	 * 
+	 * @throws CancelledException if canceled
+	 */
+	public void search(Program program, AddressSetView searchSet, TaskMonitor monitor)
+			throws CancelledException {
+		if (root == null) {
+			root = SequenceSearchState.buildStateMachine(patternList);
+		}
+
+		numToSearch = getNumToSearch(program, searchSet);
+		monitor.setMessage(searchName + " Search");
+		monitor.initialize(numToSearch);
+
+		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		for (MemoryBlock block : blocks) {
+			monitor.setProgress(numSearched);
+			// check if entire block has anything that is searchable
+			if (!block.isInitialized()) {
+				continue;
+			}
+			if (doExecutableBlocksOnly && !block.isExecute()) {
+				continue;
+			}
+			if (searchSet != null && !searchSet.isEmpty() &&
+				!searchSet.intersects(block.getStart(), block.getEnd())) {
+				continue;
+			}
+
+			try {
+				searchBlock(root, program, block, searchSet, monitor);
+			}
+			catch (IOException e) {
+				Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName);
+			}
+			numSearched += block.getSize();
+		}
+	}
+
+	private long getNumToSearch(Program program, AddressSetView searchSet) {
+		long numAddresses = 0;
+		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		for (MemoryBlock block : blocks) {
+			// check if entire block has anything that is searchable
+			if (!block.isInitialized()) {
+				continue;
+			}
+			if (doExecutableBlocksOnly && !block.isExecute()) {
+				continue;
+			}
+			if (searchSet != null && !searchSet.isEmpty() &&
+				!searchSet.intersects(block.getStart(), block.getEnd())) {
+				continue;
+			}
+			numAddresses += block.getSize();
+		}
+		return numAddresses;
+	}
+
+	/**
+	 * Search through bytes of a memory block using the finite state machine -root-
+	 * Apply any additional rules for matching patterns.
+	 * 
+	 * @param program is the Program being searched
+	 * @param block is the specific block of bytes being searched
+	 * 
+	 * @throws IOException exception during read of memory
+	 * @throws CancelledException canceled search
+	 */
+	private void searchBlock(SequenceSearchState rootState, Program program, MemoryBlock block,
+			AddressSetView restrictSet, TaskMonitor monitor)
+			throws IOException, CancelledException {
+
+		// if no restricted set, make restrict set the full block
+		AddressSet doneSet;
+		if (restrictSet == null || restrictSet.isEmpty()) {
+			doneSet = new AddressSet(block.getStart(), block.getEnd());
+		}
+		else {
+			doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd());
+		}
+
+		long numInDoneSet = doneSet.getNumAddresses();
+		long numInBlock = block.getSize();
+
+		Address blockStartAddr = block.getStart();
+
+		// pull each range off the restricted set
+		long progress = monitor.getProgress();
+		AddressRangeIterator addressRanges = doneSet.getAddressRanges();
+		long numDone = 0;
+		while (addressRanges.hasNext()) {
+			monitor.checkCanceled();
+			monitor.setMessage(searchName + " Search");
+			monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet)));
+			AddressRange addressRange = addressRanges.next();
+			long numAddressesInRange = addressRange.getLength();
+
+			ArrayList<Match> mymatches = new ArrayList<>();
+
+			long streamoffset = blockStartAddr.getOffset();
+
+			// Give block a starting/ending point before this address to search
+			//    patterns might start before, since they have a pre-pattern
+			// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
+			//       normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
+			//       normally the whole memory blocks will get searched.
+			long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
+			blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
+			if (blockOffset <= 0) {
+				// don't go before the block start
+				blockOffset = 0;
+			}
+
+			// compute number of bytes in the range + 1, and don't search more than that.
+			long maxBlockSearchLength =
+				addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
+
+			InputStream data = block.getData();
+			data.skip(blockOffset);
+
+			rootState.apply(data, maxBlockSearchLength, mymatches, monitor);
+			monitor.checkCanceled();
+
+			monitor.setMessage(searchName + " (Examine Matches)");
+
+			// TODO: DANGER there is much offset<-->address calculation here
+			//       should be OK, since they are all relative to the block.
+			long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet));
+			for (int i = 0; i < mymatches.size(); ++i) {
+				monitor.checkCanceled();
+				monitor.setProgress(
+					matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size())));
+				Match match = mymatches.get(i);
+				Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
+				if (!match.checkPostRules(streamoffset + blockOffset)) {
+					continue;
+				}
+
+				MatchAction[] matchactions = match.getMatchActions();
+				preMatchApply(matchactions, addr);
+				for (MatchAction matchaction : matchactions) {
+					matchaction.apply(program, addr, match);
+				}
+
+				postMatchApply(matchactions, addr);
+			}
+
+			numDone += numAddressesInRange;
+		}
+	}
+
+	/**
+	 * Called before any match rules are applied
+	 * @param matchactions actions that matched
+	 * @param addr address of match
+	 */
+	public void preMatchApply(MatchAction[] matchactions, Address addr) {
+		// override if any initialization needs to be done before rule application
+	}
+
+	/**
+	 * Called after any match rules are applied
+	 * Can use for cross post rule matching state application and cleanup.
+	 * @param matchactions actions that matched
+	 * @param addr adress of match
+	 */
+	public void postMatchApply(MatchAction[] matchactions, Address addr) {
+		// override if any cleanup from rule match application is needed
+	}
+}

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java

@@ -24,12 +24,22 @@ import generic.jar.ResourceFile;
 import ghidra.util.xml.SpecXmlUtils;
 import ghidra.xml.*;
 
+/**
+ * Pattern is an association of a DittedBitSequence to match,
+ * a set of post rules after a match is found that must be satisfied,
+ * and a set of actions to be taken if the pattern matches.
+ * 
+ * These patterns can be restored from an XML file.
+ */
 public class Pattern extends DittedBitSequence {
 
 	private int markOffset;	// Within pattern what is the 'marked' byte
 	private PostRule[] postrule;
 	private MatchAction[] actions;
 
+	/**
+	 * Construct an empty pattern.  Use XML to initialize
+	 */
 	public Pattern() {
 		markOffset = 0;
 		postrule = null;
@@ -37,7 +47,17 @@ public class Pattern extends DittedBitSequence {
 
 	}
 
-	public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, MatchAction[] matchArray) {
+	/**
+	 * Construct the pattern based on a DittedByteSequence a match offset, post matching rules,
+	 * and a set of actions to take when the match occurs.
+	 * 
+	 * @param seq DittedByteSequence
+	 * @param offset offset from the actual match location to report a match
+	 * @param postArray post set of rules to check for the match
+	 * @param matchArray MatchActions to apply when a match occurs
+	 */
+	public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray,
+			MatchAction[] matchArray) {
 		super(seq);
 		markOffset = offset;
 		postrule = postArray;
@@ -51,8 +71,8 @@ public class Pattern extends DittedBitSequence {
 	public MatchAction[] getMatchActions() {
 		return actions;
 	}
-	
+
-	public void setMatchActions(MatchAction[] actions){
+	public void setMatchActions(MatchAction[] actions) {
 		this.actions = actions;
 	}
 

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,8 +15,22 @@
  */
 package ghidra.util.bytesearch;
 
+/**
+ * Interface for factories that create Match Pattern classes
+ */
 public interface PatternFactory {
+	/**
+	 * Get a named match action
+	 * 
+	 * @param nm name of action to find
+	 * @return match action with the given name, null otherwise
+	 */
 	public MatchAction getMatchActionByName(String nm);
-	
+
+	/**
+	 * Get a named post match rule by name
+	 * @param nm name of the post rule
+	 * @return the post rule with the name, null otherwise
+	 */
 	public PostRule getPostRuleByName(String nm);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java

@@ -23,66 +23,104 @@ import ghidra.xml.XmlElement;
 import ghidra.xml.XmlPullParser;
 
 /**
- * Two collections of patterns that are paired together to create larger patterns
+ * A set of "pre" DittedBitSequences and a set of "post" Patterns are paired to form a larger pattern.
- * The final large patterns all must first match a pattern from the "pre" pattern collection
+ * To match, a sequence from the "pre" sequence set must first match, then one of the "post" patterns
- * followed immediately by a pattern from the "post" pattern collection
+ * is matched relative to the matching "pre" pattern.  This class is really a storage object for the
+ * patterns and provides a mechanism to read the pre/post patterns from an XML file.
  *
+ * The larger pattern has the idea of bits of check, which means the number of bits that are fixed to
+ * a value when matching (not don't care).  There is a pre pattern bits of check and post pattern bits
+ * of check.  The bits of check are used to statistically gauge the accuracy of the pattern.
+ * 
+ * An example of the XML format follows:
+ *   <patternpairs totalbits="32" postbits="16">
+ *    <prepatterns>
+ *      <data>0xe12fff1.                  </data>
+ *      <data>0xe12fff1e 0x46c0           </data>
+ *      <data>0xe12fff1e 0xe1a00000       </data>
+ *    </prepatterns>
+ *    
+ *  <postpatterns>
+ *      <data> 0xe24dd...                              11101001 00101101 .1...... ....0000  </data>
+ *      <data> 11101001 00101101 .1...... ....0000     0xe24dd...                           </data>
+ *      <data> 11101001 00101101 .1...... ....0000     0x........ 0xe24dd...                </data>
+ *      <align mark="0" bits="3"/>
+ *      <setcontext name="TMode" value="0"/>
+ *      <funcstart/>
+ *    </postpatterns>
+ *  </patternpairs>
+ *  
+ *  Note: The post Patterns can also have a set of rules that must be satisfied along with one of the
+ *  Pattern DittedBitSequence matches.
  */
 public class PatternPairSet {
 	private int totalBitsOfCheck;				// Minimum number of bits of check in final patterns
 	private int postBitsOfCheck;				// Minimum bits of check in "post" part of pattern
 	private ArrayList<DittedBitSequence> preSequences;
 	private ArrayList<Pattern> postPatterns;
-	
+
+	/**
+	 * Construct an empty PatternPairSet.  Use XML to initialize the pattern sets.
+	 */
 	public PatternPairSet() {
 		preSequences = new ArrayList<DittedBitSequence>();
 		postPatterns = new ArrayList<Pattern>();
 	}
-	
+
 	public void createFinalPatterns(ArrayList<Pattern> finalpats) {
-		for(int i=0;i<postPatterns.size();++i) {
+		for (int i = 0; i < postPatterns.size(); ++i) {
 			Pattern postpattern = postPatterns.get(i);
 			int postcheck = postpattern.getNumFixedBits();
 			if (postcheck < postBitsOfCheck) {
 				continue;
 			}
-			for(int j=0;j<preSequences.size();++j) {
+			for (DittedBitSequence prepattern : preSequences) {
-				DittedBitSequence prepattern = preSequences.get(j);
 				int precheck = prepattern.getNumFixedBits();
 				if (precheck + postcheck < totalBitsOfCheck) {
 					continue;
 				}
 				DittedBitSequence concat = prepattern.concatenate(postpattern);
-				Pattern finalpattern = new Pattern(concat,prepattern.getSize(),postpattern.getPostRules(),postpattern.getMatchActions());
+				Pattern finalpattern = new Pattern(concat, prepattern.getSize(),
+					postpattern.getPostRules(), postpattern.getMatchActions());
 				finalpats.add(finalpattern);
 			}
 		}
 	}
-	
+
+	/**
+	 * Add this PatternPairSets post patterns to an existing arraylist of patterns.
+	 * @param postpats array to add this PatternPairSets post patterns into
+	 */
 	public void extractPostPatterns(ArrayList<Pattern> postpats) {
-		for(int i=0;i<postPatterns.size();++i) {
+		for (int i = 0; i < postPatterns.size(); ++i) {
 			postpats.add(postPatterns.get(i));
 		}
 	}
-	
+
-	public void restoreXml(XmlPullParser parser,PatternFactory pfactory) throws IOException {
+	/**
+	 * Restore PatternPairSet from XML pull parser
+	 * @param parser XML pull parser
+	 * @param pfactory pattern factory user to construct patterns
+	 * @throws IOException if pull parsing fails
+	 */
+	public void restoreXml(XmlPullParser parser, PatternFactory pfactory) throws IOException {
 		XmlElement el = parser.start("patternpairs");
 		totalBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("totalbits"));
 		postBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("postbits"));
 		parser.start("prepatterns");
-		el= parser.peek();
+		el = parser.peek();
-		while(el.isStart()) {
+		while (el.isStart()) {
 			DittedBitSequence preseq = new DittedBitSequence();
 			preseq.restoreXmlData(parser);
 			preSequences.add(preseq);
 			el = parser.peek();
 		}
 		parser.end();
-		while(parser.peek().isStart()) {
+		while (parser.peek().isStart()) {
 			parser.start("postpatterns");
 			el = parser.peek();
 			ArrayList<DittedBitSequence> postdit = new ArrayList<DittedBitSequence>();
-			while(el.isStart() && el.getName().equals("data")) {
+			while (el.isStart() && el.getName().equals("data")) {
 				DittedBitSequence postseq = new DittedBitSequence();
 				postseq.restoreXmlData(parser);
 				if (postseq.getNumFixedBits() >= postBitsOfCheck) {
@@ -99,8 +137,8 @@ public class PatternPairSet {
 			postRuleArray.toArray(postRules);
 			MatchAction[] matchActions = new MatchAction[matchActionArray.size()];
 			matchActionArray.toArray(matchActions);
-			for(int i=0;i<postdit.size();++i) {
+			for (DittedBitSequence element : postdit) {
-				Pattern postpat = new Pattern(postdit.get(i),0,postRules,matchActions);
+				Pattern postpat = new Pattern(element, 0, postRules, matchActions);
 				postPatterns.add(postpat);
 			}
 			parser.end();	// End postpatterns

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,8 +17,22 @@ package ghidra.util.bytesearch;
 
 import ghidra.xml.XmlPullParser;
 
+/**
+ * Inteface for post match rules that are checked after a match is idenfied
+ */
 public interface PostRule {
-	public boolean apply(Pattern pat,long matchoffset);
+	/**
-	
+	 * Apply a post rule given the matching pattern and offset into the byte stream.
+	 * @param pat pattern that matched
+	 * @param matchoffset offset of the match
+	 * @return true if the PostRule is satisfied
+	 */
+	public boolean apply(Pattern pat, long matchoffset);
+
+	/**
+	 * Can restore state of instance PostRule from XML
+	 * 
+	 * @param parser XML pull parser
+	 */
 	public void restoreXml(XmlPullParser parser);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java

@@ -0,0 +1,466 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.util.bytesearch;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+
+import ghidra.util.task.TaskMonitor;
+
+/**
+ * SeqenceSearchState holds the state of a search for a DittedBitSequence within a byte
+ * sequence.
+ */
+public class SequenceSearchState implements Comparable<SequenceSearchState> {
+
+	private static final int PATTERN_ENDED = Integer.MAX_VALUE;
+	private SequenceSearchState parent;
+	private ArrayList<DittedBitSequence> possible;		// Patterns that could still match in this state
+	private ArrayList<DittedBitSequence> success;		// Patterns that have matched successfully if we reached this state
+	private SequenceSearchState[] trans;				// State transitions based on next byte
+
+	/**
+	 * Construct a sub sequence state with a parent sequence
+	 * 
+	 * @param parent parent SequenceSearchState
+	 */
+	public SequenceSearchState(SequenceSearchState parent) {
+		this.parent = parent;
+		possible = new ArrayList<DittedBitSequence>();
+		success = null;
+		trans = null;
+	}
+
+	/**
+	 * @return maximum number of bytes that could be matched by this sequence
+	 */
+	public int getMaxSequenceSize() {
+		int max = 0;
+		for (DittedBitSequence element : possible) {
+			int val = element.getSize();
+			if (val > max) {
+				max = val;
+			}
+		}
+		return max;
+	}
+
+	/**
+	 * Add a pattern to this search sequence.  The last pattern added is the successful
+	 * match pattern.
+	 * @param pat pattern to add
+	 * @param pos position within the current set of patterns to add this pattern
+	 */
+	public void addSequence(DittedBitSequence pat, int pos) {
+		possible.add(pat);
+		if (pos == pat.getSize()) {
+			if (success == null) {
+				success = new ArrayList<DittedBitSequence>();
+			}
+			success.add(pat);
+		}
+	}
+
+	/**
+	 * Sort the sequences that have been added 
+	 */
+	public void sortSequences() {
+		Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
+			@Override
+			public int compare(DittedBitSequence o1, DittedBitSequence o2) {
+				return o1.getIndex() - o2.getIndex();
+			}
+		};
+		Collections.sort(possible, comp);
+		if (success != null) {
+			Collections.sort(success, comp);
+		}
+	}
+
+	@Override
+	public int compareTo(SequenceSearchState o) {
+		int i = 0;
+		for (;;) {
+			if (possible.size() <= i) {
+				if (o.possible.size() <= i) {
+					return 0;
+				}
+				return -1;
+			}
+			if (o.possible.size() <= i) {
+				return 1;
+			}
+			int indus = possible.get(i).getIndex();		// Lexicographic compare an sequence of sequences
+			int indthem = o.possible.get(i).getIndex();
+			if (indus != indthem) {
+				return (indus < indthem) ? -1 : 1;
+			}
+			i += 1;
+		}
+	}
+
+	private void buildSingleTransition(ArrayList<SequenceSearchState> all, int pos, int val) {
+		SequenceSearchState newstate = null;
+		for (DittedBitSequence curpat : possible) {
+			if (curpat.isMatch(pos, val)) {
+				if (newstate == null) {
+					newstate = new SequenceSearchState(this);
+				}
+				newstate.addSequence(curpat, pos + 1);
+			}
+		}
+		trans[val] = newstate;
+		if (newstate != null) {
+			newstate.sortSequences();
+			all.add(newstate);
+		}
+	}
+
+	private void exportSuccess(ArrayList<Match> match, int offset) {
+		for (DittedBitSequence succes : success) {		// If we found matches
+			Match newmatch = new Match(succes, offset);
+			match.add(newmatch);
+		}
+	}
+
+	/**
+	 * Merge in -op- and this as a single state
+	 * @param op
+	 */
+	private void merge(SequenceSearchState op) {
+		SequenceSearchState parent = op.parent;
+		for (int i = 0; i < 256; ++i) {
+			if (parent.trans[i] == op) {
+				parent.trans[i] = this;			// Should be replaced with this
+			}
+		}
+		if (op.success != null) {				// Merge 
+			if (success == null) {
+				success = op.success;
+			}
+			else {
+				ArrayList<DittedBitSequence> tmp = new ArrayList<DittedBitSequence>();
+				int i = 0;
+				int j = 0;
+				int curpat = -1;
+				int thispat = success.get(i).getIndex();
+				int oppat = op.success.get(j).getIndex();
+				while ((i < success.size()) || (j < op.success.size())) {
+					if (thispat == oppat) {
+						if (curpat != thispat) {
+							tmp.add(success.get(i));
+							curpat = thispat;
+						}
+						i += 1;
+						j += 1;
+						thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
+						oppat =
+							(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
+					}
+					else if (thispat < oppat) {
+						if (curpat != thispat) {
+							tmp.add(success.get(i));
+							curpat = thispat;
+						}
+						i += 1;
+						thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
+					}
+					else {
+						if (curpat != oppat) {
+							tmp.add(op.success.get(j));
+							curpat = oppat;
+						}
+						j += 1;
+						oppat =
+							(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
+					}
+				}
+				success = tmp;
+			}
+		}
+	}
+
+	/**
+	 * Try to match this Sequence to the byteArray, and add any matches to the match list
+	 * @param bytearray array of bytes to match
+	 * @param numbytes retrict number of bytes to allow to match
+	 * @param match list of matches, the result
+	 */
+	public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList<Match> match) {
+		int subindex = 0;
+		SequenceSearchState curstate = this;
+
+		do {
+			if (curstate.success != null) {
+				curstate.exportSuccess(match, 0);
+			}
+			if (subindex >= numbytes) {
+				return;
+			}
+			curstate = curstate.trans[0xff & bytearray[subindex]];		// Perform state transition based on next byte in buffer
+			subindex += 1;
+		}
+		while (curstate != null);
+
+	}
+
+	/**
+	 * Search for patterns in a byte array.  All matches are returned.
+	 * @param buffer is the array of bytes to search
+	 * @param match is populated with a Match object for each pattern and position that matches 
+	 */
+	public void apply(byte[] buffer, ArrayList<Match> match) {
+		SequenceSearchState curstate;
+		int subindex;
+		for (int offset = 0; offset < buffer.length; ++offset) {
+			curstate = this;			// New starting offset -> Root state
+			subindex = offset;
+			do {
+				if (curstate.success != null) {
+					curstate.exportSuccess(match, offset);
+				}
+				if (subindex >= buffer.length) {	// if we've run out of bytes, must restart at next offset
+					break;
+				}
+				curstate = curstate.trans[0xff & buffer[subindex]];	// Perform state transition based on next byte
+				subindex += 1;
+			}
+			while (curstate != null);
+		}
+	}
+
+	/**
+	 * Search for pattern in the stream -in-.
+	 * @param in - The stream to scan for matches
+	 * @param match - Any matches are appended as Match records to this ArrayList
+	 * @param monitor - if non-null, check for user cancel, and maintain progress info
+	 * @throws IOException
+	 */
+	public void apply(InputStream in, ArrayList<Match> match, TaskMonitor monitor)
+			throws IOException {
+		apply(in, -1L, match, monitor);
+	}
+
+	/**
+	 * Search for pattern in the stream -in-.
+	 * @param in - The stream to scan for matches
+	 * @param maxBytes - The maximum number of bytes to scan forward in this stream
+	 * @param match - Any matches are appended as Match records to this ArrayList
+	 * @param monitor - if non-null, check for user cancel, and maintain progress info
+	 * @throws IOException
+	 */
+	public void apply(InputStream in, long maxBytes, ArrayList<Match> match, TaskMonitor monitor)
+			throws IOException {
+		long progress = monitor.getProgress();
+
+		int maxSize = getMaxSequenceSize() + 1;
+		if (maxSize < 4096) {
+			maxSize = 4096;
+		}
+		if (maxBytes > 0) {
+			maxBytes += getMaxSequenceSize() + 1;
+		}
+		byte[] firstBuf = new byte[maxSize];
+		byte[] secondBuf = new byte[maxSize];
+		byte[] curBuf;
+		SequenceSearchState curState;
+		int fullBuffers;				// Number of buffers that are completely full
+		int ra = in.read(firstBuf);
+		if (ra == firstBuf.length) {
+			ra = in.read(secondBuf);
+			if (ra == secondBuf.length) {
+				fullBuffers = 2;
+			}
+			else {
+				if (ra < 0) {
+					ra = 0;
+				}
+				fullBuffers = 1;
+				byte[] tmp = new byte[ra];
+				for (int i = 0; i < ra; ++i) {
+					tmp[i] = secondBuf[i];
+				}
+				secondBuf = tmp;
+			}
+		}
+		else if (ra < 0) {
+			return;				// No bytes at all were read
+		}
+		else {
+			byte[] tmp = new byte[ra];
+			for (int i = 0; i < ra; ++i) {
+				tmp[i] = firstBuf[i];
+			}
+			firstBuf = tmp;
+			fullBuffers = 0;
+			secondBuf = new byte[0];
+		}
+		int offset = 0;
+		int bufRelativeOffset = 0;
+		int subIndex;
+		while (fullBuffers == 2) {
+			curState = this;			// New starting offset -> Root state
+			subIndex = bufRelativeOffset;
+			curBuf = firstBuf;
+			do {
+				if (curState.success != null) {
+					curState.exportSuccess(match, offset);
+				}
+				if (subIndex >= curBuf.length) {						// check that we have enough bytes in current buffer
+					curBuf = secondBuf;									// If not, switch to secondary buffer
+					subIndex = 0;
+				}
+				curState = curState.trans[0xff & curBuf[subIndex]];		// Perform state transition based on next byte in buffer
+				subIndex += 1;
+			}
+			while (curState != null);
+			offset += 1;												// Advance to next starting offset
+			if (maxBytes > 0 && offset > maxBytes) {
+				break;
+			}
+			bufRelativeOffset += 1;
+			if (bufRelativeOffset == firstBuf.length) {							// If starting offset no longer falls in firstbuf
+				byte[] tmp = firstBuf;									//     Switch firstbuf with secondbuf
+				firstBuf = secondBuf;
+				secondBuf = tmp;
+				ra = in.read(secondBuf);							//     refill secondbuf (old firstbuf) with new bytes
+				if (monitor != null) {
+					if (monitor.isCancelled()) {
+						return;
+					}
+					monitor.setProgress(progress + offset);
+				}
+				if (ra != secondBuf.length) {
+					fullBuffers = 1;
+					if (ra < 0) {
+						ra = 0;
+					}
+					tmp = new byte[ra];
+					for (int i = 0; i < ra; ++i) {
+						tmp[i] = secondBuf[i];
+					}
+					secondBuf = tmp;
+				}
+				bufRelativeOffset = 0;
+			}
+		}
+
+		while (fullBuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
+			if (secondBuf.length == 0) {
+				fullBuffers = 0;
+			}
+			curState = this;
+			subIndex = bufRelativeOffset;
+			curBuf = firstBuf;
+			do {
+				if (curState.success != null) {
+					curState.exportSuccess(match, offset);
+				}
+				if (subIndex >= curBuf.length) {
+					if (curBuf == secondBuf) {
+						break;				// Out of data, all pending patterns fail
+					}
+					curBuf = secondBuf;
+					subIndex = 0;
+					if (curBuf.length == 0) {
+						break;
+					}
+				}
+				curState = curState.trans[0xff & curBuf[subIndex]];
+				subIndex += 1;
+			}
+			while (curState != null);
+			offset += 1;
+			bufRelativeOffset += 1;
+			if (bufRelativeOffset == firstBuf.length) {
+				if (fullBuffers == 0) {
+					break;
+				}
+				firstBuf = secondBuf;
+				fullBuffers = 0;
+				bufRelativeOffset = 0;
+				secondBuf = new byte[0];
+			}
+		}
+	}
+
+	/**
+	 * Build a new transition level for the state machine
+	 * 
+	 * @param prev previous search sequences
+	 * @param pos position within the search sequence state for this level
+	 * @return list of possible new search states to be added to the state machine
+	 */
+	static ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,
+			int pos) {
+		ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
+		Iterator<SequenceSearchState> iterator = prev.iterator();
+		while (iterator.hasNext()) {			// For each current state
+			SequenceSearchState next = iterator.next();
+			next.trans = new SequenceSearchState[256];
+			for (int i = 0; i < 256; ++i) {		// Try every byte transition
+				next.buildSingleTransition(res, pos, i);
+			}
+		}
+		if (res.isEmpty()) {
+			return res;
+		}
+		// Prepare to dedup the states
+		Collections.sort(res);
+		ArrayList<SequenceSearchState> finalres = new ArrayList<SequenceSearchState>();
+		Iterator<SequenceSearchState> iter = res.iterator();
+		SequenceSearchState curpat = iter.next();
+		finalres.add(curpat);
+		while (iter.hasNext()) {
+			SequenceSearchState nextpat = iter.next();
+			int comp = curpat.compareTo(nextpat);
+			if (comp == 0) {		// Identical states
+				curpat.merge(nextpat);
+			}
+			else {
+				curpat = nextpat;
+				finalres.add(curpat);
+			}
+		}
+		return finalres;
+	}
+
+	/**
+	 * Build a search state machine from a list of DittedBitSequences
+	 * @param patterns bit sequence patterns
+	 * @return search state the will match the given sequences
+	 */
+	static public SequenceSearchState buildStateMachine(
+			ArrayList<? extends DittedBitSequence> patterns) {
+		SequenceSearchState root = new SequenceSearchState(null);
+		for (int i = 0; i < patterns.size(); ++i) {
+			DittedBitSequence pat = patterns.get(i);
+			pat.setIndex(i);
+			root.addSequence(pat, 0);
+		}
+		root.sortSequences();
+		ArrayList<SequenceSearchState> statelevel = new ArrayList<SequenceSearchState>();
+		statelevel.add(root);
+		int level = 0;
+		do {
+			statelevel = buildTransitionLevel(statelevel, level);
+			level += 1;
+		}
+		while (!statelevel.isEmpty());
+		return root;
+	}
+}

Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java

@@ -15,8 +15,6 @@
  */
 package ghidra.app.analyzers;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -51,7 +49,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 	private static final String DESCRIPTION =
 		"Search for architecture specific byte patterns: typically starts of functions";
 	private static final String PRE_FUNCTION_MATCH_PROPERTY_NAME = "PreFunctionMatch";
-	private static final int RESTRICTED_PATTERN_BYTE_RANGE = 32;
 	private final static String OPTION_NAME_DATABLOCKS = "Search Data Blocks";
 	private static final String OPTION_DESCRIPTION_DATABLOCKS =
 		"Search for byte patterns in blocks that are not executable";
@@ -214,7 +211,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 			applyActionToSet(program, addr, funcResult, match);
 		}
 
-
 		protected boolean checkPreRequisites(Program program, Address addr) {
 			/**
 			 * If the match's mark point occurs in undefined data, schedule disassembly
@@ -254,10 +250,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 					return false;
 				}
 			}
-			
+
 			return true;
 		}
-		
 
 		protected void applyActionToSet(Program program, Address addr, AddressSet resultSet,
 				Match match) {
@@ -632,25 +627,34 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 			return false;
 		}
 
-		AddressSet restrictedSet = removeNotSearchedAddresses(program, set);
+		boolean doExecutableBlocksOnly = checkForExecuteBlock(program) && executableBlocksOnly;
 
 		// clear out any previous potential matches, because we are re-looking at these places
 		//   this will keep cruft from accumulating in the property map.
-		getOrCreatePotentialMatchPropertyMap(program).remove(restrictedSet);
+		getOrCreatePotentialMatchPropertyMap(program).remove(set);
 
-		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		MemoryBytePatternSearcher patternSearcher;
-		for (MemoryBlock block2 : blocks) {
+		patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) {
-			MemoryBlock block = block2;
+
-			if (!restrictedSet.intersects(block.getStart(), block.getEnd())) {
+			@Override
-				continue;
+			public void preMatchApply(MatchAction[] actions, Address addr) {
+				contextValueList = null; // make sure, only context from these actions used
 			}
-			try {
+
-				searchBlock(root, program, block, restrictedSet, monitor);
+			@Override
+			public void postMatchApply(MatchAction[] actions, Address addr) {
+				// Actions might have set context, check if postcondition failed first
+				if (!postreqFailedResult.contains(addr)) {
+					setCurrentContext(program, addr);
+				}
+				// get rid of the context list.
+				contextValueList = null;
 			}
-			catch (IOException e) {
+		};
-				log.appendMsg("Unable to scan block " + block.getName() + " for function starts");
+		patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly);
-			}
+
-		}
+		patternSearcher.search(program, set, monitor);
+
 		AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program);
 		if (!disassemResult.isEmpty()) {
 			analysisManager.disassemble(disassemResult);
@@ -708,34 +712,17 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 	}
 
 	/**
-	 * Get rid of any blocks from the address set that shouldn't be searched.
+	 * @return true - if there are any blocks marked executable
-	 *
-	 * @param program
-	 * @param bset
-	 * @return
 	 */
-	private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) {
+	private boolean checkForExecuteBlock(Program program) {
-		AddressSet restrictedSet = new AddressSet(bset);
 		MemoryBlock[] blocks = program.getMemory().getBlocks();
-		boolean hasExecutable = false;
+
 		for (MemoryBlock block : blocks) {
 			if (block.isExecute()) {
-				hasExecutable = true;
+				return true;
 			}
 		}
-		for (MemoryBlock block : blocks) {
+		return false;
-			if (!block.isInitialized()) {
-				restrictedSet.deleteRange(block.getStart(), block.getEnd());
-				continue;
-			}
-			if (executableBlocksOnly && hasExecutable) {
-				if (!block.isExecute()) {
-					restrictedSet.deleteRange(block.getStart(), block.getEnd());
-					continue;
-				}
-			}
-		}
-		return restrictedSet;
 	}
 
 	@Override
@@ -816,94 +803,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 		return patlist;
 	}
 
-	/**
-	 * Search through bytes of a memory block using the finite state machine -root-
-	 * Apply any additional rules for matching patterns.
-	 * @param program is the Program being searched
-	 * @param block is the specific block of bytes being searched
-	 * @throws IOException
-	 * @throws CancelledException
-	 */
-	private void searchBlock(SequenceSearchState root, Program program, MemoryBlock block,
-			AddressSetView restrictSet, TaskMonitor monitor)
-			throws IOException, CancelledException {
-
-		// if no restricted set, make restrict set the full block
-		AddressSet doneSet = new AddressSet(restrictSet);
-		if (doneSet.isEmpty()) {
-			doneSet.addRange(block.getStart(), block.getEnd());
-		}
-		doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
-
-		Address blockStartAddr = block.getStart();
-
-		// pull each range off the restricted set
-		AddressRangeIterator addressRanges = doneSet.getAddressRanges();
-		while (addressRanges.hasNext()) {
-			monitor.checkCanceled();
-			AddressRange addressRange = addressRanges.next();
-
-			monitor.setMessage("Function Search");
-			monitor.initialize(doneSet.getNumAddresses());
-			monitor.setProgress(0);
-
-			ArrayList<Match> mymatches = new ArrayList<>();
-
-			long streamoffset = blockStartAddr.getOffset();
-
-			// Give block a starting/ending point before this address to search
-			//    patterns might start before, since they have a pre-pattern
-			// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
-			//       normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
-			//       normally the whole memory blocks will get searched.
-			long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
-			blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
-			if (blockOffset <= 0) {
-				// don't go before the block start
-				blockOffset = 0;
-			}
-
-			// compute number of bytes in the range + 1, and don't search more than that.
-			long maxBlockSearchLength =
-				addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
-
-			InputStream data = block.getData();
-			data.skip(blockOffset);
-
-			root.apply(data, maxBlockSearchLength, mymatches, monitor);
-			monitor.checkCanceled();
-
-			monitor.setMessage("Function Search (Examine Matches)");
-			monitor.initialize(mymatches.size());
-			monitor.setProgress(0);
-
-			// TODO: DANGER there is much offset<-->address calculation here
-			//       should be OK, since they are all relative to the block.
-			for (int i = 0; i < mymatches.size(); ++i) {
-				monitor.checkCanceled();
-				monitor.setProgress(i);
-				Match match = mymatches.get(i);
-				Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
-				if (!match.checkPostRules(streamoffset + blockOffset)) {
-					continue;
-				}
-				MatchAction[] matchactions = match.getMatchActions();
-				contextValueList = null; // make sure, only context from these actions used
-				for (MatchAction matchaction : matchactions) {
-					matchaction.apply(program, addr, match);
-				}
-				// Actions might have set context, check if postcondition failed first
-				if (!postreqFailedResult.contains(addr)) {
-					setCurrentContext(program, addr);
-				}
-				else {
-					// didn't apply it, get rid of the context list.
-					contextValueList = null;
-				}
-			}
-		}
-	}
-
 	@Override
 	public MatchAction getMatchActionByName(String nm) {
 		if (nm.equals("funcstart")) {

Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java

@@ -1,55 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *      http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.util.bytesearch;
-
-import ghidra.util.xml.SpecXmlUtils;
-import ghidra.xml.XmlElement;
-import ghidra.xml.XmlPullParser;
-
-public class AlignRule implements PostRule {
-
-	private int mark;		// Position, relative to start of pattern, to check alignment at
-	private int alignmask;  // Mask of bits that must be zero
-	
-	
-	public AlignRule(){
-	}
-	
-	public AlignRule(int mark, int alignmask){
-		this.mark = mark;
-		this.alignmask = alignmask;
-	}
-	
-	@Override
-	public boolean apply(Pattern pat, long matchoffset) {
-		int off = (int)matchoffset;
-		return (((off + mark) & alignmask)==0);
-	}
-
-	@Override
-	public void restoreXml(XmlPullParser parser) {
-		XmlElement el = parser.start("align");
-		mark = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
-		int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
-		alignmask = (1<<bits) - 1;
-		parser.end();
-	}
-
-	public int getAlignMask() {
-		return alignmask;
-	}
-
-}

Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java

@@ -1,385 +0,0 @@
-/* ###
- * IP: GHIDRA
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- *      http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package ghidra.util.bytesearch;
-
-import ghidra.util.task.TaskMonitor;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-
-public class SequenceSearchState implements Comparable<SequenceSearchState> {
-	
-	private SequenceSearchState parent;
-	private ArrayList<DittedBitSequence> possible;		// Patterns that could still match in this state
-	private ArrayList<DittedBitSequence> success;		// Patterns that have matched successfully if we reached this state
-	private SequenceSearchState[] trans;				// State transitions based on next byte
-	
-	public SequenceSearchState(SequenceSearchState par) {
-		parent = par;
-		possible = new ArrayList<DittedBitSequence>();
-		success = null;
-		trans = null;
-	}
-
-	public int getMaxSequenceSize() {
-		int max = 0;
-		for(int i=0;i<possible.size();++i) {
-			int val = possible.get(i).getSize();
-			if (val > max)
-				max = val;
-		}
-		return max;
-	}
-	
-	public void addSequence(DittedBitSequence pat,int pos) {
-		possible.add(pat);
-		if (pos == pat.getSize()) {
-			if (success == null)
-				success = new ArrayList<DittedBitSequence>();
-			success.add(pat);
-		}
-	}
-	
-	public void sortSequences() {
-		Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
-			@Override
-			public int compare(DittedBitSequence o1, DittedBitSequence o2) {
-				return o1.getIndex() - o2.getIndex();
-			}
-		};
-		Collections.sort(possible,comp);
-		if (success != null)
-			Collections.sort(success,comp);
-	}
-	
-	@Override
-	public int compareTo(SequenceSearchState o) {
-		int i=0;
-		for(;;) {
-			if (possible.size() <= i) {
-				if (o.possible.size() <=i)
-					return 0;
-				return -1;
-			}
-			if (o.possible.size() <= i)
-				return 1;
-			int indus = possible.get(i).getIndex();		// Lexicographic compare an sequence of sequences
-			int indthem = o.possible.get(i).getIndex();
-			if (indus != indthem)
-				return (indus < indthem) ? -1 : 1;
-			i += 1;
-		}
-	}
-	
-	private void buildSingleTransition(ArrayList<SequenceSearchState> all,int pos,int val) {
-		SequenceSearchState newstate = null;
-		for(int i=0;i<possible.size();++i) {
-			DittedBitSequence curpat = possible.get(i);
-			if (curpat.isMatch(pos, val)) {
-				if (newstate == null)
-					newstate = new SequenceSearchState(this);
-				newstate.addSequence(curpat, pos+1);
-			}
-		}
-		trans[val] = newstate;
-		if (newstate != null) {
-			newstate.sortSequences();
-			all.add(newstate);
-		}
-	}
-	
-	private void exportSuccess(ArrayList<Match> match,int offset) {
-		for(int i=0;i<success.size();++i) {		// If we found matches
-			Match newmatch = new Match(success.get(i),offset);
-			match.add(newmatch);
-		}			
-	}
-	
-	/**
-	 * Merge in -op- and this as a single state
-	 * @param op
-	 */
-	private void merge(SequenceSearchState op) {
-		SequenceSearchState parent = op.parent;
-		for(int i=0;i<256;++i) {
-			if (parent.trans[i] == op)			// Any references to -op- in parent
-				parent.trans[i] = this;			// Should be replaced with this
-		}
-		if (op.success != null) {				// Merge 
-			if (success == null) {
-				success = op.success;
-			}
-			else {
-				ArrayList<DittedBitSequence> tmp = new ArrayList<DittedBitSequence>();
-				int i=0;
-				int j=0;
-				int curpat = -1;
-				int thispat = success.get(i).index;
-				int oppat = op.success.get(j).index;
-				while((i<success.size())||(j<op.success.size())) {
-					if (thispat == oppat) {
-						if (curpat != thispat) {
-							tmp.add(success.get(i));
-							curpat = thispat;
-						}
-						i += 1;
-						j += 1;
-						thispat = (i==success.size()) ? 10000000 : success.get(i).index;
-						oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
-					}
-					else if (thispat < oppat) {
-						if (curpat != thispat) {
-							tmp.add(success.get(i));
-							curpat = thispat;
-						}
-						i += 1;
-						thispat = (i==success.size()) ? 10000000 : success.get(i).index;
-					}
-					else {
-						if (curpat != oppat) {
-							tmp.add(op.success.get(j));
-							curpat = oppat;
-						}
-						j += 1;
-						oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
-					}
-				}
-				success = tmp;
-			}
-		}
-	}
-	
-	public void sequenceMatch(byte[] bytearray,int numbytes,ArrayList<Match> match) {
-		int subindex = 0;
-		SequenceSearchState curstate = this;
-		
-		do {
-			if (curstate.success != null)
-				curstate.exportSuccess(match, 0);
-			if (subindex >= numbytes) return;
-			curstate = curstate.trans[ 0xff & bytearray[subindex] ];		// Perform state transition based on next byte in buffer
-			subindex += 1;
-		} while(curstate != null);
-		
-	}
-
-	/**
-	 * Search for patterns in a byte array.  All matches are returned.
-	 * @param buffer is the array of bytes to search
-	 * @param match is populated with a Match object for each pattern and position that matches 
-	 */
-	public void apply(byte[] buffer,ArrayList<Match> match) {
-		SequenceSearchState curstate;
-		int subindex;
-		for(int offset=0;offset<buffer.length;++offset) {
-			curstate = this;			// New starting offset -> Root state
-			subindex = offset;
-			do {
-				if (curstate.success != null)		// Check for any successful pattern matches for bytes up to this point
-					curstate.exportSuccess(match, offset);
-				if (subindex >= buffer.length) {	// if we've run out of bytes, must restart at next offset
-					break;
-				}
-				curstate = curstate.trans[ 0xff & buffer[subindex] ];	// Perform state transition based on next byte
-				subindex += 1;
-			} while(curstate != null);
-		}		
-	}
-
-	/**
-	 * Search for pattern in the stream -in-.
-	 * @param in - The stream to scan for matches
-	 * @param match - Any matches are appended as Match records to this ArrayList
-	 * @param monitor - if non-null, check for user cancel, and maintain progress info
-	 * @throws IOException
-	 */
-	public void apply(InputStream in,ArrayList<Match> match,TaskMonitor monitor) throws IOException {
-		apply(in,-1L,match,monitor);
-	}
-	
-	/**
-	 * Search for pattern in the stream -in-.
-	 * @param in - The stream to scan for matches
-	 * @param maxBytes - The maximum number of bytes to scan forward in this stream
-	 * @param match - Any matches are appended as Match records to this ArrayList
-	 * @param monitor - if non-null, check for user cancel, and maintain progress info
-	 * @throws IOException
-	 */
-	public void apply(InputStream in, long maxBytes, ArrayList<Match> match,TaskMonitor monitor) throws IOException {
-		int maxsize = getMaxSequenceSize()+1;
-		if (maxsize <4096)
-			maxsize = 4096;
-		if (maxBytes > 0) {
-			maxBytes += getMaxSequenceSize()+1;
-		}
-		byte[] firstbuf=new byte[maxsize];
-		byte[] secondbuf=new byte[maxsize];
-		byte[] curbuf;
-		SequenceSearchState curstate;
-		int fullbuffers;				// Number of buffers that are completely full
-		int ra = in.read(firstbuf);
-		if (ra == firstbuf.length) {
-			ra = in.read(secondbuf);
-			if (ra == secondbuf.length) {
-				fullbuffers = 2;
-			}
-			else {
-				if (ra < 0)
-					ra = 0;
-				fullbuffers = 1;
-				byte[] tmp = new byte[ra];
-				for(int i=0;i<ra;++i)
-					tmp[i] = secondbuf[i];
-				secondbuf = tmp;
-			}
-		}
-		else if (ra < 0)
-			return;				// No bytes at all were read
-		else {
-			byte[] tmp = new byte[ra];
-			for(int i=0;i<ra;++i)
-				tmp[i] = firstbuf[i];
-			firstbuf = tmp;
-			fullbuffers = 0;
-			secondbuf = new byte[0];
-		}
-		int offset=0;
-		int bufreloff=0;
-		int subindex;
-		while(fullbuffers == 2) {
-			curstate = this;			// New starting offset -> Root state
-			subindex = bufreloff;
-			curbuf = firstbuf;
-			do {
-				if (curstate.success != null)		// Check for any successful pattern matches for bytes up to this point
-					curstate.exportSuccess(match, offset);
-				if (subindex >= curbuf.length) {						// check that we have enough bytes in current buffer
-					curbuf = secondbuf;									// If not, switch to secondary buffer
-					subindex = 0;
-				}
-				curstate = curstate.trans[ 0xff & curbuf[subindex] ];		// Perform state transition based on next byte in buffer
-				subindex += 1;
-			} while(curstate != null);
-			offset += 1;												// Advance to next starting offset
-			if (maxBytes > 0 && offset > maxBytes) {
-				break;
-			}
-			bufreloff += 1;
-			if (bufreloff == firstbuf.length) {							// If starting offset no longer falls in firstbuf
-				byte[] tmp = firstbuf;									//     Switch firstbuf with secondbuf
-				firstbuf = secondbuf;
-				secondbuf = tmp;
-				ra = in.read(secondbuf);							//     refill secondbuf (old firstbuf) with new bytes
-				if (monitor!=null) {
-					if (monitor.isCancelled()) return;
-					monitor.setProgress(offset);
-				}
-				if (ra != secondbuf.length) {
-					fullbuffers = 1;
-					if (ra < 0)
-						ra = 0;
-					tmp = new byte[ra];
-					for(int i=0;i<ra;++i)
-						tmp[i] = secondbuf[i];
-					secondbuf = tmp;
-				}
-				bufreloff = 0;
-			}
-		}
-		
-		while(fullbuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
-			if (secondbuf.length == 0)
-				fullbuffers = 0;
-			curstate = this;
-			subindex = bufreloff;
-			curbuf = firstbuf;
-			do {
-				if (curstate.success != null)
-					curstate.exportSuccess(match, offset);
-				if (subindex >= curbuf.length) {
-					if (curbuf == secondbuf) break;				// Out of data, all pending patterns fail
-					curbuf = secondbuf;
-					subindex = 0;
-					if (curbuf.length==0) break;
-				}
-				curstate = curstate.trans[ 0xff & curbuf[subindex] ];
-				subindex += 1;
-			} while(curstate != null);
-			offset += 1;
-			bufreloff += 1;
-			if (bufreloff == firstbuf.length) {
-				if (fullbuffers == 0) break;
-				firstbuf = secondbuf;
-				fullbuffers = 0;
-				bufreloff = 0;
-				secondbuf = new byte[0];
-			}
-		}
-	}
-	
-	static public ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,int pos) {
-		ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
-		Iterator<SequenceSearchState> iterator = prev.iterator();
-		while(iterator.hasNext()) {			// For each current state
-			SequenceSearchState next = iterator.next();
-			next.trans = new SequenceSearchState[256];
-			for(int i=0;i<256;++i) {		// Try every byte transition
-				next.buildSingleTransition(res, pos, i);
-			}
-		}
-		if (res.isEmpty()) return res;
-		// Prepare to dedup the states
-		Collections.sort(res);
-		ArrayList<SequenceSearchState> finalres = new ArrayList<SequenceSearchState>();
-		Iterator<SequenceSearchState> iter = res.iterator();
-		SequenceSearchState curpat = iter.next();
-		finalres.add(curpat);
-		while(iter.hasNext()) {
-			SequenceSearchState nextpat = iter.next();
-			int comp = curpat.compareTo(nextpat);
-			if (comp == 0) {		// Identical states
-				curpat.merge(nextpat);
-			}
-			else {
-				curpat = nextpat;
-				finalres.add(curpat);
-			}
-		}
-		return finalres;
-	}
-	
-	static public SequenceSearchState buildStateMachine(ArrayList<? extends DittedBitSequence> patterns) {
-		SequenceSearchState root = new SequenceSearchState(null);
-		for(int i=0;i<patterns.size();++i) {
-			DittedBitSequence pat = patterns.get(i);
-			pat.index = i;
-			root.addSequence(pat, 0);
-		}
-		root.sortSequences();
-		ArrayList<SequenceSearchState> statelevel = new ArrayList<SequenceSearchState>();
-		statelevel.add(root);
-		int level = 0;
-		do {
-			statelevel = buildTransitionLevel(statelevel, level);
-			level += 1;				
-		} while(!statelevel.isEmpty());
-		return root;
-	}
-}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java

@@ -38,7 +38,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 		extends BackgroundCommand {
 
 	protected final String name;
-	protected final Address address;
+	private Address address;
 	protected final int count;
 	protected final DataValidationOptions validationOptions;
 	protected final DataApplyOptions applyOptions;
@@ -141,7 +141,8 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 	 * @return true if the data type creation completes successfully.
 	 * @throws CancelledException if the user cancels this task.
 	 */
-	private boolean doApplyTo(Program program, TaskMonitor taskMonitor) throws CancelledException {
+	protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
+			throws CancelledException {
 
 		try {
 			monitor = taskMonitor;
@@ -353,4 +354,24 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 		return (applyOptions.shouldClearDefinedData()) ? ClearDataMode.CLEAR_ALL_CONFLICT_DATA
 				: ClearDataMode.CLEAR_ALL_UNDEFINED_CONFLICT_DATA;
 	}
+
+	/**
+	 * Get the address for the data item to be processed by the base implementation.
+	 * In general this is the initial model address set when the command was created.
+	 * 
+	 * @return the address of the data item being created.
+	 */
+	final protected Address getDataAddress() {
+		return address;
+	}
+
+	/**
+	 * Set the address of the data item to be applied.
+	 * Can be used for sub classes that need to apply multiple data items.
+	 * 
+	 * @param addr set the current data address
+	 */
+	final protected void setDataAddress(Address addr) {
+		address = addr;
+	}
 }

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/CreateTypeDescriptorBackgroundCmd.java

@@ -73,7 +73,7 @@ public class CreateTypeDescriptorBackgroundCmd
 
 	private void loadModel(Program program) {
 		if (model == null || program != model.getProgram()) {
-			model = new TypeDescriptorModel(program, address, validationOptions);
+			model = new TypeDescriptorModel(program, getDataAddress(), validationOptions);
 		}
 	}
 
@@ -138,13 +138,13 @@ public class CreateTypeDescriptorBackgroundCmd
 		String prefix = demangledName + " ";
 
 		// Plate Comment
-		EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, address,
+		EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
 			applyOptions);
 
 		monitor.checkCanceled();
 
 		// Label
-		EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, address,
+		EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
 			applyOptions);
 
 		return true;

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHCatchHandlerMapBackgroundCmd.java

@@ -74,7 +74,7 @@ public class CreateEHCatchHandlerMapBackgroundCmd
 	@Override
 	protected EHCatchHandlerModel createModel(Program program) {
 		if (model == null) {
-			model = new EHCatchHandlerModel(program, count, address, validationOptions);
+			model = new EHCatchHandlerModel(program, count, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHESTypeListBackgroundCmd.java

@@ -70,7 +70,7 @@ public class CreateEHESTypeListBackgroundCmd
 	@Override
 	protected EHESTypeListModel createModel(Program program) {
 		if (model == null) {
-			model = new EHESTypeListModel(program, address, validationOptions);
+			model = new EHESTypeListModel(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHFuncInfoBackgroundCmd.java

@@ -71,7 +71,7 @@ public class CreateEHFuncInfoBackgroundCmd
 	@Override
 	protected EHFunctionInfoModel createModel(Program program) {
 		if (model == null) {
-			model = new EHFunctionInfoModel(program, address, validationOptions);
+			model = new EHFunctionInfoModel(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHIPToStateMapBackgroundCmd.java

@@ -72,7 +72,7 @@ public class CreateEHIPToStateMapBackgroundCmd
 	@Override
 	protected EHIPToStateModel createModel(Program program) {
 		if (model == null) {
-			model = new EHIPToStateModel(program, count, address, validationOptions);
+			model = new EHIPToStateModel(program, count, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHTryBlockMapBackgroundCmd.java

@@ -73,7 +73,7 @@ public class CreateEHTryBlockMapBackgroundCmd
 	@Override
 	protected EHTryBlockModel createModel(Program program) {
 		if (model == null) {
-			model = new EHTryBlockModel(program, count, address, validationOptions);
+			model = new EHTryBlockModel(program, count, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHUnwindMapBackgroundCmd.java

@@ -73,7 +73,7 @@ public class CreateEHUnwindMapBackgroundCmd extends AbstractCreateDataBackground
 	@Override
 	protected EHUnwindModel createModel(Program program) {
 		if (model == null) {
-			model = new EHUnwindModel(program, count, address, validationOptions);
+			model = new EHUnwindModel(program, count, getDataAddress(), validationOptions);
 		}
 		return model;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti1BackgroundCmd.java

@@ -63,7 +63,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 	@Override
 	protected Rtti1Model createModel(Program program) {
 		if (model == null || program != model.getProgram()) {
-			model = new Rtti1Model(program, address, validationOptions);
+			model = new Rtti1Model(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}
@@ -101,14 +101,14 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 			}
 			catch (InvalidDataTypeException e) {
 				// Couldn't get pmd and attributes so leave it off and simply log the error.
-				String message = "Unable to get PMD and attributes for RTTI1 at " + address + ".";
+				String message = "Unable to get PMD and attributes for RTTI1 at " + getDataAddress() + ".";
 				handleError(message);
 			}
 
 			// Plate Comment
 			EHDataTypeUtilities.createPlateCommentIfNeeded(program,
 				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
-				RTTI_1_NAME, suffix, address, applyOptions);
+				RTTI_1_NAME, suffix, getDataAddress(), applyOptions);
 
 			monitor.checkCanceled();
 
@@ -116,7 +116,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 			if (applyOptions.shouldCreateLabel()) {
 				String rtti1Suffix = RTTI_1_NAME + suffix;
 				rtti1Suffix = SymbolUtilities.replaceInvalidChars(rtti1Suffix, true);
-				RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, rtti1Suffix);
+				RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, rtti1Suffix);
 			}
 
 		}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti2BackgroundCmd.java

@@ -68,7 +68,7 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 	@Override
 	protected Rtti2Model createModel(Program program) {
 		if (model == null || program != model.getProgram()) {
-			model = new Rtti2Model(program, rtti1Count, address, validationOptions);
+			model = new Rtti2Model(program, rtti1Count, getDataAddress(), validationOptions);
 		}
 		return model;
 	}
@@ -118,13 +118,13 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 		// Plate Comment
 		EHDataTypeUtilities.createPlateCommentIfNeeded(program,
 			RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
-			RTTI_2_NAME, null, address, applyOptions);
+			RTTI_2_NAME, null, getDataAddress(), applyOptions);
 
 		monitor.checkCanceled();
 
 		// Label
 		if (applyOptions.shouldCreateLabel()) {
-			RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_2_NAME);
+			RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_2_NAME);
 		}
 		return true;
 	}

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti3BackgroundCmd.java

@@ -63,7 +63,7 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 	@Override
 	protected Rtti3Model createModel(Program program) {
 		if (model == null || program != model.getProgram()) {
-			model = new Rtti3Model(program, address, validationOptions);
+			model = new Rtti3Model(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}
@@ -105,13 +105,13 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 			// Plate Comment
 			EHDataTypeUtilities.createPlateCommentIfNeeded(program,
 				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
-				RTTI_3_NAME, null, address, applyOptions);
+				RTTI_3_NAME, null, getDataAddress(), applyOptions);
 
 			monitor.checkCanceled();
 
 			// Label
 			if (applyOptions.shouldCreateLabel()) {
-				RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_3_NAME);
+				RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_3_NAME);
 			}
 		}
 

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti4BackgroundCmd.java

@@ -15,29 +15,32 @@
  */
 package ghidra.app.cmd.data.rtti;
 
-import java.util.List;
+import java.util.*;
-import java.util.Set;
 
 import ghidra.app.cmd.data.*;
 import ghidra.app.util.datatype.microsoft.DataApplyOptions;
 import ghidra.app.util.datatype.microsoft.DataValidationOptions;
 import ghidra.program.model.address.Address;
+import ghidra.program.model.address.AddressSet;
 import ghidra.program.model.data.InvalidDataTypeException;
 import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.MemoryBlock;
 import ghidra.program.model.symbol.Namespace;
 import ghidra.program.util.ProgramMemoryUtil;
+import ghidra.util.bytesearch.*;
 import ghidra.util.exception.CancelledException;
+import ghidra.util.task.TaskMonitor;
 
 /**
- * This command will create an RTTI4 data type. 
+ * This command will create multiple RTTI4 data types all at one time. 
- * If there are any existing instructions in the area to be made into data, the command will fail.
+ * If there are any existing instructions in the areas to be made into data, the command will fail only on that RTTI4 entry.
  * Any data in the area will be replaced with the new dataType.
  */
 public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rtti4Model> {
 
 	private static final String RTTI_4_NAME = "RTTI Complete Object Locator";
 	private List<MemoryBlock> vfTableBlocks;
+	private List<Address> rtti4Locations;
 
 	/**
 	 * Constructs a command for applying an RTTI4 dataType at an address.
@@ -54,12 +57,44 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 
 		super(Rtti4Model.DATA_TYPE_NAME, address, 1, validationOptions, applyOptions);
 		this.vfTableBlocks = vfTableBlocks;
+		rtti4Locations = new ArrayList<Address>();
+		rtti4Locations.add(address);
+	}
+
+	public CreateRtti4BackgroundCmd(List<Address> addresses, List<MemoryBlock> vfTableBlocks,
+			DataValidationOptions validationOptions, DataApplyOptions applyOptions) {
+
+		super(Rtti4Model.DATA_TYPE_NAME, addresses.get(0), 1, validationOptions, applyOptions);
+		this.rtti4Locations = addresses;
+		this.vfTableBlocks = vfTableBlocks;
+	}
+
+	@Override
+	protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
+			throws CancelledException {
+
+		// process each potential RTTI4 entry, keeping track of good RTTI4 tables
+		List<Address> goodRtti4Locations = new ArrayList<Address>();
+		boolean succeeded = false;
+		for (Address addr : rtti4Locations) {
+			setDataAddress(addr);
+			succeeded |= super.doApplyTo(program, taskMonitor);
+			goodRtti4Locations.add(addr);
+		}
+
+		// if any succeeded and should create associated data, make the vftables all at one time
+		if (succeeded && applyOptions.shouldFollowData()) {
+			createAssociatedVfTables(program, goodRtti4Locations, taskMonitor);
+		}
+
+		return succeeded;
 	}
 
 	@Override
 	protected Rtti4Model createModel(Program program) {
-		if (model == null || program != model.getProgram()) {
+		if (model == null || program != model.getProgram() ||
-			model = new Rtti4Model(program, address, validationOptions);
+			!getDataAddress().equals(model.getAddress())) {
+			model = new Rtti4Model(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}
@@ -75,7 +110,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 		}
 		catch (InvalidDataTypeException e) {
 			createRtti0Success = false;
-			// log message and continue with other markup.
+			// log message and continue with other mark-up.
 			handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
 		}
 
@@ -89,9 +124,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 			handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
 		}
 
-		boolean createVfTableSuccess = createVfTable();
+		return createRtti0Success && createRtti3Success;
-
-		return createRtti0Success && createRtti3Success && createVfTableSuccess;
 	}
 
 	private boolean createRtti0() throws CancelledException, InvalidDataTypeException {
@@ -112,57 +145,95 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 		return cmd.applyTo(model.getProgram(), monitor);
 	}
 
-	private boolean createVfTable() throws CancelledException {
+	private boolean createAssociatedVfTables(Program program, List<Address> goodRtti4Locations,
+			TaskMonitor taskMonitor) throws CancelledException {
 
-		monitor.checkCanceled();
+		MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables");
 
-		Program program = model.getProgram();
+		HashMap<Address, VfTableModel> foundVFtables = new HashMap<>();
 
-		Address rtti4Address = address;
+		for (Address rtti4Address : goodRtti4Locations) {
-		int defaultPointerSize = program.getDefaultPointerSize();
-		int alignment = defaultPointerSize; // Align the vf table based on the size of the pointers in it.
-		Set<Address> directRtti4Refs =
-			ProgramMemoryUtil.findDirectReferences(program, vfTableBlocks, alignment, rtti4Address,
-				monitor);
 
-		VfTableModel validVfTableModel = null;
+			byte[] bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti4Address);
-		for (Address possibleVfMetaAddr : directRtti4Refs) {
 
+			addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
+		}
+
+		AddressSet searchSet = new AddressSet();
+		for (MemoryBlock block : vfTableBlocks) {
+			searchSet.add(block.getStart(), block.getEnd());
+		}
+
+		searcher.search(program, searchSet, monitor);
+
+		// did the search, now process the results
+		boolean didSome = false;
+		for (Address rtti4Address : goodRtti4Locations) {
 			monitor.checkCanceled();
 
-			Address possibleVfTableAddr = possibleVfMetaAddr.add(defaultPointerSize);
+			VfTableModel vfTableModel = foundVFtables.get(rtti4Address);
+			if (vfTableModel == null) {
+				String message =
+					"No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
+				handleErrorMessage(program, rtti4Address, message);
+				continue;
+			}
 
-			// Validate the model. Don't apply the command if invalid.
+			CreateVfTableBackgroundCmd cmd =
-			try {
+				new CreateVfTableBackgroundCmd(vfTableModel, applyOptions);
-				VfTableModel vfTableModel =
+			didSome |= cmd.applyTo(program, monitor);
-					new VfTableModel(program, possibleVfTableAddr, validationOptions);
+		}
-				vfTableModel.validate();
 
-				if (validVfTableModel != null) {
+		return didSome;
-					String message = "More than one possible vfTable found for " +
+	}
-						Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
+
-					handleErrorMessage(program, rtti4Address, message);
+	/**
-					return false;
+	 * Add a search pattern, to the searcher, for the set of bytes representing an rtti4 location.
+	 * Only one VFTable for is allowed for an RTT4 location, last one in wins and gets created.
+	 * 
+	 * @param searcher byte pattern searcher
+	 * @param foundVFtables list of addresses accumulated when actual search is performed
+	 * @param rtti4Address location of rttiAddress to find vfTable for
+	 * @param bytes bytes representing rtti4Addres to be found in memory
+	 */
+	private void addByteSearchPattern(MemoryBytePatternSearcher searcher,
+			HashMap<Address, VfTableModel> foundVFtables, Address rtti4Address, byte[] bytes) {
+
+		if (bytes == null) {
+			return;
+		}
+
+		GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti4Address) {
+			@Override
+			public void apply(Program prog, Address addr, Match match) {
+
+				Address possibleVfTableAddr = addr.add(prog.getDefaultPointerSize());
+
+				// See if VfTable is valid, and add to rtti4 to vfTable map
+				try {
+					VfTableModel vfTableModel =
+						new VfTableModel(prog, possibleVfTableAddr, validationOptions);
+					vfTableModel.validate();
+
+					VfTableModel existing = foundVFtables.put(rtti4Address, vfTableModel);
+
+					if (existing != null) {
+						// potential table already found, is an error, don't know which is right
+						String message = "More than one possible vfTable found for " +
+							Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
+						handleErrorMessage(prog, rtti4Address, message);
+					}
+				}
+				catch (InvalidDataTypeException e) {
+					// This isn't a valid model.
 				}
-				validVfTableModel = vfTableModel;
 			}
-			catch (InvalidDataTypeException e) {
+		};
-				continue; // This isn't a valid model.
-			}
-		}
 
-		if (validVfTableModel == null) {
+		GenericByteSequencePattern<Address> genericByteMatchPattern =
-			String message =
+			new GenericByteSequencePattern<Address>(bytes, action);
-				"No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
-			handleErrorMessage(program, rtti4Address, message);
-			return false;
-		}
 
-		monitor.checkCanceled();
+		searcher.addPattern(genericByteMatchPattern);
-
-		CreateVfTableBackgroundCmd cmd =
-			new CreateVfTableBackgroundCmd(validVfTableModel, applyOptions);
-		return cmd.applyTo(program, monitor);
 	}
 
 	@Override
@@ -177,20 +248,20 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 
 		if (rtti0Model != null) {
 
+			// Plate Comment
 			// Plate Comment
 			EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
-				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
+				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER, RTTI_4_NAME,
-				RTTI_4_NAME, null, address, applyOptions);
+				null, getDataAddress(), applyOptions);
-
 			monitor.checkCanceled();
 
 			// Label
 			if (applyOptions.shouldCreateLabel()) {
-				RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_4_NAME);
+				RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model,
+					RTTI_4_NAME);
 			}
 		}
 
 		return true;
 	}
-
 }

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateVfTableBackgroundCmd.java

@@ -15,7 +15,7 @@
  */
 package ghidra.app.cmd.data.rtti;
 
-import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.getAbsoluteAddress;
+import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.*;
 
 import ghidra.app.cmd.data.*;
 import ghidra.app.util.datatype.microsoft.DataApplyOptions;
@@ -70,7 +70,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 	@Override
 	protected VfTableModel createModel(Program program) {
 		if (model == null || program != model.getProgram()) {
-			model = new VfTableModel(program, address, validationOptions);
+			model = new VfTableModel(program, getDataAddress(), validationOptions);
 		}
 		return model;
 	}
@@ -105,7 +105,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 			return false;
 		}
 		long displacement = dataType.getLength();
-		Address terminatorAddress = address.add(displacement);
+		Address terminatorAddress = getDataAddress().add(displacement);
 		try {
 			Address referencedAddress = getAbsoluteAddress(program, terminatorAddress);
 			if (referencedAddress == null || referencedAddress.getOffset() != 0) {
@@ -136,7 +136,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 	private boolean createMetaPointer() {
 
 		Program program = model.getProgram();
-		Address metaAddress = address.subtract(program.getDefaultPointerSize());
+		Address metaAddress = getDataAddress().subtract(program.getDefaultPointerSize());
 
 		// Create a pointer to the RTTI4 associated with the vf table.
 		DataType metaPointer = new PointerDataType(program.getDataTypeManager());
@@ -162,7 +162,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 
 	private boolean createVfTableMarkup() throws CancelledException, InvalidDataTypeException {
 
-		Address vfTableAddress = address;
+		Address vfTableAddress = getDataAddress();
 		Program program = model.getProgram();
 
 		monitor.checkCanceled();
@@ -172,9 +172,8 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 		if (rtti0Model != null) {
 
 			// Plate Comment
-			EHDataTypeUtilities.createPlateCommentIfNeeded(program,
+			EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
-				RttiUtil.CONST_PREFIX + RttiUtil.getDescriptorTypeNamespace(rtti0Model) +
+				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
-					Namespace.DELIMITER,
 				VF_TABLE_LABEL, null, vfTableAddress, applyOptions);
 
 			monitor.checkCanceled();
@@ -188,7 +187,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 
 		// Create functions that are referred to by the vf table.
 		if (applyOptions.shouldCreateFunction()) {
-			int elementCount = model.getElementCount();
+			int elementCount = model.getCount();
 			for (int tableElementIndex = 0; tableElementIndex < elementCount; tableElementIndex++) {
 				monitor.checkCanceled();
 				Address vfPointer = model.getVirtualFunctionPointer(tableElementIndex);
@@ -238,7 +237,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
 	 * the vftable.
 	 */
 	private Address getMetaAddress(Program program) {
-		return address.subtract(program.getDefaultPointerSize());
+		return getDataAddress().subtract(program.getDefaultPointerSize());
 	}
 
 }

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/VfTableModel.java

@@ -15,7 +15,7 @@
  */
 package ghidra.app.cmd.data.rtti;
 
-import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.getAbsoluteAddress;
+import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.*;
 
 import ghidra.app.cmd.data.AbstractCreateDataTypeModel;
 import ghidra.app.cmd.data.TypeDescriptorModel;
@@ -40,7 +40,6 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
 
 	private DataType dataType;
 	private Rtti4Model rtti4Model;
-	private int elementCount = -1;
 
 	private Program lastProgram;
 	private DataType lastDataType;
@@ -80,7 +79,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
 		long entrySize = individualEntryDataType.getLength();
 
 		// Each entry is a pointer to where a function can possibly be created.
-		long numEntries = RttiUtil.getVfTableCount(program, startAddress);
+		long numEntries = getCount();
 		if (numEntries == 0) {
 			throw new InvalidDataTypeException(
 				getName() + " data type at " + getAddress() + " doesn't have a valid vf table.");
@@ -123,7 +122,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
 			lastDataType = null;
 			lastElementCount = -1;
 
-			lastElementCount = RttiUtil.getVfTableCount(program, getAddress());
+			lastElementCount = getCount();
 			if (lastElementCount > 0) {
 				DataTypeManager dataTypeManager = program.getDataTypeManager();
 				PointerDataType pointerDt = new PointerDataType(dataTypeManager);
@@ -168,17 +167,6 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
 		return getAbsoluteAddress(getProgram(), address);
 	}
 
-	/**
-	 * Gets the number of elements in the vf table. Returns 0 if this model isn't for a valid vf table.
-	 * @return the number of vf table elements or 0.
-	 */
-	public int getElementCount() {
-		if (elementCount == -1) {
-			elementCount = RttiUtil.getVfTableCount(getProgram(), getAddress());
-		}
-		return elementCount;
-	}
-
 	/**
 	 * Gets the type descriptor (RTTI 0) model associated with this vf table.
 	 * @return the type descriptor (RTTI 0) model or null.

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java

@@ -30,6 +30,7 @@ import ghidra.program.model.lang.UndefinedValueException;
 import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.MemoryBlock;
 import ghidra.program.util.ProgramMemoryUtil;
+import ghidra.util.bytesearch.*;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
@@ -41,7 +42,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
 
 	private static final String NAME = "Windows x86 PE RTTI Analyzer";
 	private static final String DESCRIPTION =
-		"This analyzer finds and creates all of the RTTI metadata structures and their associated vf tables.";
+		"Finds and creates RTTI metadata structures and associated vf tables.";
 
 	// TODO If we want the RTTI analyzer to find all type descriptors regardless of whether
 	//      they are used for RTTI, then change the CLASS_PREFIX_CHARS to ".". Need to be
@@ -133,6 +134,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
 		monitor.setMaximum(possibleRtti0Addresses.size());
 		monitor.setMessage("Creating RTTI Data...");
 
+		ArrayList<Address> rtti0Locations = new ArrayList<Address>();
 		int count = 0;
 		for (Address rtti0Address : possibleRtti0Addresses) {
 			monitor.checkCanceled();
@@ -157,29 +159,29 @@ public class RttiAnalyzer extends AbstractAnalyzer {
 				rtti0Address, validationOptions, applyOptions);
 			typeDescCmd.applyTo(program, monitor);
 
-			// Create any valid RTTI4s for this TypeDescriptor
+			rtti0Locations.add(rtti0Address);
-			processRtti4sForRtti0(program, rtti0Address, monitor);
 		}
+
+		// Create any valid RTTI4s for this TypeDescriptor
+		processRtti4sForRtti0(program, rtti0Locations, monitor);
 	}
 
-	private void processRtti4sForRtti0(Program program, Address rtti0Address, TaskMonitor monitor)
+	private void processRtti4sForRtti0(Program program, List<Address> rtti0Locations,
-			throws CancelledException {
+			TaskMonitor monitor) throws CancelledException {
 
-		List<MemoryBlock> rDataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
+		List<MemoryBlock> dataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
 			program.getMemory(), ".rdata", monitor);
 
+		dataBlocks.addAll(ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
+			program.getMemory(), ".data", monitor));
+
 		List<Address> rtti4Addresses =
-			getRtti4Addresses(program, rDataBlocks, rtti0Address, validationOptions, monitor);
+			getRtti4Addresses(program, dataBlocks, rtti0Locations, validationOptions, monitor);
 
-		for (Address rtti4Address : rtti4Addresses) {
+		// create all found RTTI4 tables at once
-
+		CreateRtti4BackgroundCmd cmd = new CreateRtti4BackgroundCmd(rtti4Addresses, dataBlocks,
-			monitor.checkCanceled();
+			validationOptions, applyOptions);
-
+		cmd.applyTo(program, monitor);
-			CreateRtti4BackgroundCmd cmd =
-				new CreateRtti4BackgroundCmd(rtti4Address, rDataBlocks, validationOptions,
-					applyOptions);
-			cmd.applyTo(program, monitor);
-		}
 	}
 
 	/**
@@ -187,67 +189,128 @@ public class RttiAnalyzer extends AbstractAnalyzer {
 	 * the RTTI 0 at the indicated base address.
 	 * @param program the program containing the RTTI 0 data structure.
 	 * @param rtti4Blocks the memory blocks to be searched for RTTI4 structures.
-	 * @param rtti0Address the base address of the RTTI 0 structure in the program
+	 * @param rtti0Locations the base addresses of the RTTI 0 structure in the program
 	 * @param validationOptions options indicating how validation is performed for data structures
-	 * @param monitor the task monitor for cancelling a task
+	 * @param monitor the task monitor for canceling a task
 	 * @return the RTTI 4 base addresses associated with the RTTI 0
 	 * @throws CancelledException if the user cancels this task.
 	 */
 	private static List<Address> getRtti4Addresses(Program program, List<MemoryBlock> rtti4Blocks,
-			Address rtti0Address, DataValidationOptions validationOptions, TaskMonitor monitor)
+			List<Address> rtti0Locations, DataValidationOptions validationOptions,
-			throws CancelledException {
+			TaskMonitor monitor) throws CancelledException {
 
 		monitor.checkCanceled();
 
-		List<Address> addresses = new ArrayList<>(); // the RTTI 4 addresses
+		List<Address> addresses =
-		int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset();
+			getRefsToRtti0(program, rtti4Blocks, rtti0Locations, validationOptions, monitor);
-		Set<Address> refsToRtti0 = getRefsToRtti0(program, rtti4Blocks, rtti0Address);
 
-		// for each RTTI 0 now see if we can get RTTI4s that refer to it.
-		for (Address refAddress : refsToRtti0) {
-
-			monitor.checkCanceled();
-
-			Address possibleRtti4Address;
-			try {
-				possibleRtti4Address = refAddress.subtractNoWrap(rtti0PointerOffset);
-			}
-			catch (AddressOverflowException e) {
-				continue; // Couldn't get an Rtti4 address.
-			}
-
-			Rtti4Model rtti4Model =
-				new Rtti4Model(program, possibleRtti4Address, validationOptions);
-			try {
-				rtti4Model.validate();
-			}
-			catch (InvalidDataTypeException e) {
-				continue; // Only process valid RTTI 4 data.
-			}
-
-			// Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly 
-			// through the RTTI 3.
-			boolean refersToRtti0 = rtti4Model.refersToRtti0(rtti0Address);
-			if (!refersToRtti0) {
-				continue; // Only process valid RTTI 4 data.
-			}
-
-			addresses.add(possibleRtti4Address);
-		}
 		return addresses;
 	}
 
-	private static Set<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
+	/** For each of the RTTI0 locations found locate the associated RTTI4 structure referring to it.
-			Address rtti0Address) throws CancelledException {
+	 * 
-		Set<Address> refsToRtti0;
+	 * @param program program to be searched
-		if (MSDataTypeUtils.is64Bit(program)) {
+	 * @param dataBlocks dataBlocks to search
-			refsToRtti0 = ProgramMemoryUtil.findImageBaseOffsets32(program, 4, rtti0Address,
+	 * @param rtti0Locations list of known rtti0 locations
-				TaskMonitor.DUMMY);
+	 * @param validationOptions options for validation of found RTTI4 entries
+	 * @param monitor to cancel
+	 * @return list of found RTTI4 references to known RTTI0 locations
+	 * @throws CancelledException if canceled
+	 */
+	private static List<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
+			List<Address> rtti0Locations, DataValidationOptions validationOptions,
+			TaskMonitor monitor) throws CancelledException {
+
+		List<Address> addresses = new ArrayList<>(); // the RTTI 4 addresses
+
+		int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset();
+
+		MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI0 refernces");
+
+		for (Address rtti0Address : rtti0Locations) {
+			byte[] bytes;
+			if (MSDataTypeUtils.is64Bit(program)) {
+				// 64-bit programs will have the addresses as offsets from the image base (BOS)
+				bytes = ProgramMemoryUtil.getImageBaseOffsets32Bytes(program, 4, rtti0Address);
+
+				addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
+					rtti0Address, bytes);
+			}
+			else {
+				// 32-bit could have direct address in memory
+				bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti0Address);
+
+				addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
+					rtti0Address, bytes);
+			}
 		}
-		else {
+
-			refsToRtti0 = ProgramMemoryUtil.findDirectReferences(program, dataBlocks,
+		AddressSet searchSet = new AddressSet();
-				program.getDefaultPointerSize(), rtti0Address, TaskMonitor.DUMMY);
+		for (MemoryBlock block : dataBlocks) {
+			searchSet.add(block.getStart(), block.getEnd());
 		}
-		return refsToRtti0;
+
+		searcher.search(program, searchSet, monitor);
+
+		return addresses;
+	}
+
+	/**
+	 * Add a search pattern, to the searcher, for the set of bytes representing an address
+	 * 
+	 * @param searcher pattern searcher
+	 * @param validationOptions RTTI4 validation options
+	 * @param addresses list of found valid RTTI4 locations accumulated during actual search
+	 * @param rtti0PointerOffset offset of pointer in RTTI4 entry to RTTI0
+	 * @param rtti0Address RTTI0 address corresponding to pattern of bytes
+	 * @param bytes pattern of bytes in memory corresponding to address
+	 */
+	private static void addByteSearchPattern(MemoryBytePatternSearcher searcher,
+			DataValidationOptions validationOptions, List<Address> addresses,
+			int rtti0PointerOffset, Address rtti0Address, byte[] bytes) {
+
+		// no pattern bytes.
+		if (bytes == null) {
+			return;
+		}
+
+		// Each time a match for this byte pattern validate as an RTTI4 and add to list
+		GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti0Address) {
+			@Override
+			public void apply(Program prog, Address addr, Match match) {
+				Address possibleRtti4Address;
+				try {
+					possibleRtti4Address = addr.subtractNoWrap(rtti0PointerOffset);
+				}
+				catch (AddressOverflowException e) {
+					return; // Couldn't get an Rtti4 address.
+				}
+
+				Rtti4Model rtti4Model =
+					new Rtti4Model(prog, possibleRtti4Address, validationOptions);
+				try {
+					rtti4Model.validate();
+				}
+				catch (InvalidDataTypeException e) {
+					return; // Only process valid RTTI 4 data.
+				}
+
+				// Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly 
+				// through the RTTI 3.
+				boolean refersToRtti0 = rtti4Model.refersToRtti0(getMatchValue());
+				if (!refersToRtti0) {
+					return; // Only process valid RTTI 4 data.
+				}
+
+				// add to list of RTTI4 locations to be processed later
+				addresses.add(possibleRtti4Address);
+			}
+		};
+
+		// create a Pattern of the bytes and the MatchAction to perform upon a match
+		GenericByteSequencePattern<Address> genericByteMatchPattern =
+			new GenericByteSequencePattern<Address>(bytes, action);
+
+		searcher.addPattern(genericByteMatchPattern);
 	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/util/AddressSetPropertyMapDB.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +15,10 @@
  */
 package ghidra.program.database.util;
 
+import java.util.ConcurrentModificationException;
+
+import db.*;
+import db.util.ErrorHandler;
 import ghidra.program.database.ProgramDB;
 import ghidra.program.database.map.AddressMap;
 import ghidra.program.model.address.*;
@@ -26,11 +29,6 @@ import ghidra.util.exception.CancelledException;
 import ghidra.util.exception.DuplicateNameException;
 import ghidra.util.task.TaskMonitor;
 
-import java.util.ConcurrentModificationException;
-
-import db.*;
-import db.util.ErrorHandler;
-
 /**
  * AddressSetPropertyMap that uses a RangeMapDB to maintain a set of addresses.
  * 
@@ -74,8 +72,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 			DBHandle dbh = program.getDBHandle();
 			String tableName = AddressSetPropertyMapDB.TABLE_PREFIX + mapName;
 			if (dbh.getTable(tableName) != null) {
-				throw new DuplicateNameException("Address Set Property Map named " + mapName +
+				throw new DuplicateNameException(
-					" already exists.");
+					"Address Set Property Map named " + mapName + " already exists.");
 			}
 
 			return new AddressSetPropertyMapDB(program, mapName, program, addrMap, lock);
@@ -91,9 +89,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 		this.mapName = mapName;
 		this.lock = lock;
 
-		propertyMap =
+		propertyMap = new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
-			new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
+			program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
-				program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
 	}
 
 	@Override
@@ -114,7 +111,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#add(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void add(AddressSet addressSet) {
+	public void add(AddressSetView addressSet) {
 		checkDeleted();
 
 		lock.acquire();
@@ -135,7 +132,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#set(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void set(AddressSet addressSet) {
+	public void set(AddressSetView addressSet) {
 		checkDeleted();
 		lock.acquire();
 		try {
@@ -169,7 +166,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#remove(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void remove(AddressSet addressSet) {
+	public void remove(AddressSetView addressSet) {
 		checkDeleted();
 		lock.acquire();
 		try {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/util/AddressSetPropertyMap.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,59 +22,60 @@ import ghidra.program.model.address.*;
  *
  */
 public interface AddressSetPropertyMap {
-	
+
 	/**
 	 * Add the address range to the property map.
 	 * @param start start of the range
 	 * @param end end of the range
 	 */
 	void add(Address start, Address end);
-	
+
 	/**
 	 * Add the address set to the property map.
 	 * @param addressSet address set to add
 	 */
-	void add(AddressSet addressSet);
+	void add(AddressSetView addressSet);
-	
+
 	/**
 	 * Clear the property map and set it with the given address set. 
 	 * @param addressSet address set to use
 	 */
-	void set(AddressSet addressSet);
+	void set(AddressSetView addressSet);
+
 	/**
 	 * Remove the address range from the property map.
 	 * @param start start of the range
 	 * @param end end of the range
 	 */
 	void remove(Address start, Address end);
-	
+
 	/**
 	 * Remove the address set from the property map.
 	 * @param addressSet address set to remove
 	 */
-	void remove(AddressSet addressSet);
+	void remove(AddressSetView addressSet);
-	
+
 	/**
 	 * Return the address set for the property map.
 	 */
 	AddressSet getAddressSet();
-	
+
 	/**
 	 * Return an address iterator over the property map. 
 	 */
 	AddressIterator getAddresses();
-	
+
 	/**
 	 * Return an address range iterator over the property map.
 	 */
 	AddressRangeIterator getAddressRanges();
-	
+
 	/**
 	 * Clear the property map.
 	 *
 	 */
 	void clear();
-	
+
 	/**
 	 * Return whether the property map contains the given address.
 	 * @param addr address to check