GT-3341_emteere_RTTIPerformance code review changes, refactored progress

monitor

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java

@@ -20,20 +20,11 @@ import ghidra.xml.XmlElement;
 import ghidra.xml.XmlPullParser;
 
 /**
- * ByteSearch post search rule when a pattern is found, The pattern must have a certain
+ * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
- * alignment at an offset from the location the pattern matches.  The alignment is
+ * alignment at an offset from the location the pattern matches. 
- * specified by the mask bits that must be zero.
  * 
- *   mark is the offset in bytes from the start of the matching pattern.
+ * The pattern can be constructed or restored from XML of the form,
- *   
+ * where alignOffset=mark, alignmask=bits
- *   align 2 = 0x1 - lower bit must be zero
- *   align 4 = 0x3 - lower two bits must be zero
- *   align 8 = 0x7 - lower three bits must be zero
- *   align 16 = 0xF - lower four bits must be zero
- *   ....
- * Other strange alignments could be specified, but most likely the above suffice.
- * 
- * The pattern can be constructed or restored from XML of the form:
  * 
  *     <align mark="0" bits="1"/>
  *   
@@ -41,27 +32,44 @@ import ghidra.xml.XmlPullParser;
 
 public class AlignRule implements PostRule {
 
-	private int mark;		// Position, relative to start of pattern, to check alignment at
+	private int alignOffset;  // Position, relative to start of pattern, to check alignment at
-	private int alignmask;  // Mask of bits that must be zero
+	private int alignmask;    // Mask of bits that must be zero
 
 	public AlignRule() {
 	}
 
-	public AlignRule(int mark, int alignmask) {
+	/**
-		this.mark = mark;
+	 * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
+	 * alignment at an offset from the location the pattern matches. The alignment is
+	 * specified by the alignmask bits that must be zero.
+	 * 
+	 *   Normally alignOffset is 0, since most patterns will match at the address that must be aligned
+	 *   To align a match, use the following
+	 *
+	 *  align to  2 = alignmask 0x1 - lower bit must be zero
+	 *  align to  4 = alignmask 0x3 - lower two bits must be zero
+	 *  align to  8 = alignmask 0x7 - lower three bits must be zero
+	 *  align to 16 = alignmask 0xF - lower four bits must be zero
+	 *  ....
+	 *  Other strange alignments could be specified, but most likely the above suffice.
+	 * @param alignOffset - bytes offset from pattern to check for alignment
+	 * @param alignmask - the mask where a 1 bit must be zero
+	 */
+	public AlignRule(int alignOffset, int alignmask) {
+		this.alignOffset = alignOffset;
 		this.alignmask = alignmask;
 	}
 
 	@Override
 	public boolean apply(Pattern pat, long matchoffset) {
 		int off = (int) matchoffset;
-		return (((off + mark) & alignmask) == 0);
+		return (((off + alignOffset) & alignmask) == 0);
 	}
 
 	@Override
 	public void restoreXml(XmlPullParser parser) {
 		XmlElement el = parser.start("align");
-		mark = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
+		alignOffset = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
 		int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
 		alignmask = (1 << bits) - 1;
 		parser.end();

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java

@@ -29,6 +29,13 @@ import ghidra.program.model.data.DataType;
 
 public class GenericByteSequencePattern<T> extends Pattern {
 
+	/**
+	 * Construct a sequence of bytes with no mask, and associated action
+	 * to be called if this pattern matches.
+	 * 
+	 * @param bytesSequence sequence of bytes to match
+	 * @param action action to apply if the match succeeds
+	 */
 	public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction<T> action) {
 		super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]);
 
@@ -36,6 +43,14 @@ public class GenericByteSequencePattern<T> extends Pattern {
 		matchActions[0] = action;
 	}
 
+	/**
+	 * Construct a sequence of bytes with a mask, and associated action
+	 * to be called if this pattern matches.
+	 * 
+	 * @param bytesSequence sequence of bytes to match
+	 * @param mask mask, bits that are 1 must match the byteSequence bits
+	 * @param action to apply if the match succeeds
+	 */
 	public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask,
 			GenericMatchAction<DataType> action) {
 		super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]);

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java

@@ -26,8 +26,9 @@ package ghidra.util.bytesearch;
 public class GenericMatchAction<T> extends DummyMatchAction {
 	T matchValue;
 
-	/*
+	/**
-	 * construct with an appropriate object that can be used when the action is applied
+	 * Construct a match action used when a match occurs for some GenericByteSequece
+	 * @param matchValue specialized object used when match occurs
 	 */
 	public GenericMatchAction(T matchValue) {
 		this.matchValue = matchValue;

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java

@@ -24,9 +24,15 @@ package ghidra.util.bytesearch;
  * 
  */
 public class Match {
-	private DittedBitSequence sequence;	// Pattern that matches
+	private DittedBitSequence sequence;	// Pattern that matched
-	private long offset;			// starting offset within bytestream of match
+	private long offset;			    // Offset within bytestream where the match occurred
 
+	/**
+	 * Construct a Match of a DittedBitSequence at an offset within a byte stream.
+	 * Object normally used when a match occurs during a MemoryBytePatternSearch.
+	 * @param sequence that matched
+	 * @param offset from the start of byte stream where the matched occured
+	 */
 	public Match(DittedBitSequence sequence, long offset) {
 		this.sequence = sequence;
 		this.offset = offset;

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java

@@ -23,7 +23,19 @@ import ghidra.xml.XmlPullParser;
  * Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern
  */
 public interface MatchAction {
+	/**
+	 * Apply the match action to the program at the address.
+	 * 
+	 * @param program program in which the match occurred
+	 * @param addr where the match occured
+	 * @param match information about the match that occurred
+	 */
 	public void apply(Program program, Address addr, Match match);
 
+	/**
+	 * Action can be constructed from XML
+	 * 
+	 * @param parser XML pull parser to restore action from XML
+	 */
 	public void restoreXml(XmlPullParser parser);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java

@@ -28,6 +28,7 @@ import ghidra.util.task.TaskMonitor;
 
 /**
  * Multi pattern/mask/action memory searcher
+ * Patterns must be supplied/added, or a pre-initialized searchState supplied
  * 
  * Preload search patterns and actions, then call search method.
  */
@@ -35,13 +36,20 @@ import ghidra.util.task.TaskMonitor;
 public class MemoryBytePatternSearcher {
 	private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;
 
+	SequenceSearchState root = null;
+
 	ArrayList<Pattern> patternList;
 
 	private String searchName = "";
 
+	private boolean doExecutableBlocksOnly = false;  // only search executable blocks
+
+	private long numToSearch = 0;
+	private long numSearched = 0;
+
 	/**
 	 * Create with pre-created patternList
-	 * 
+	 * @param searchName name of search
 	 * @param patternList - list of patterns(bytes/mask/action)
 	 */
 	public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {
@@ -49,8 +57,19 @@ public class MemoryBytePatternSearcher {
 		this.patternList = new ArrayList<Pattern>(patternList);
 	}
 
+	/**
+	 * Create with an initialized SequenceSearchState
+	 * @param searchName name of search
+	 * @param root search state pre-initialized
+	 */
+	public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) {
+		this.searchName = searchName;
+		this.root = root;
+	}
+
 	/**
 	 * Create with no patternList, must add patterns before searching
+	 * @param searchName name of search
 	 * 
 	 */
 	public MemoryBytePatternSearcher(String searchName) {
@@ -66,6 +85,10 @@ public class MemoryBytePatternSearcher {
 		patternList.add(pattern);
 	}
 
+	public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) {
+		this.doExecutableBlocksOnly = doExecutableBlocksOnly;
+	}
+
 	/**
 	 * Search initialized memory blocks for all patterns(bytes/mask/action).
 	 * Call associated action for each pattern matched.
@@ -78,23 +101,59 @@ public class MemoryBytePatternSearcher {
 	 */
 	public void search(Program program, AddressSetView searchSet, TaskMonitor monitor)
 			throws CancelledException {
-		SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList);
+		if (root == null) {
+			root = SequenceSearchState.buildStateMachine(patternList);
+		}
+
+		numToSearch = getNumToSearch(program, searchSet);
+		monitor.setMessage(searchName + " Search");
+		monitor.initialize(numToSearch);
 
 		MemoryBlock[] blocks = program.getMemory().getBlocks();
-		for (MemoryBlock block2 : blocks) {
+		for (MemoryBlock block : blocks) {
-			MemoryBlock block = block2;
+			monitor.setProgress(numSearched);
-			if (!searchSet.intersects(block.getStart(), block.getEnd())) {
+			// check if entire block has anything that is searchable
+			if (!block.isInitialized()) {
 				continue;
 			}
+			if (doExecutableBlocksOnly && !block.isExecute()) {
+				continue;
+			}
+			if (searchSet != null && !searchSet.isEmpty() &&
+				!searchSet.intersects(block.getStart(), block.getEnd())) {
+				continue;
+			}
+
 			try {
 				searchBlock(root, program, block, searchSet, monitor);
 			}
 			catch (IOException e) {
-				Msg.error(this, "Unable to scan block " + block.getName() + " for patterns");
+				Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName);
 			}
+			numSearched += block.getSize();
 		}
 	}
 
+	private long getNumToSearch(Program program, AddressSetView searchSet) {
+		long numAddresses = 0;
+		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		for (MemoryBlock block : blocks) {
+			// check if entire block has anything that is searchable
+			if (!block.isInitialized()) {
+				continue;
+			}
+			if (doExecutableBlocksOnly && !block.isExecute()) {
+				continue;
+			}
+			if (searchSet != null && !searchSet.isEmpty() &&
+				!searchSet.intersects(block.getStart(), block.getEnd())) {
+				continue;
+			}
+			numAddresses += block.getSize();
+		}
+		return numAddresses;
+	}
+
 	/**
 	 * Search through bytes of a memory block using the finite state machine -root-
 	 * Apply any additional rules for matching patterns.
@@ -110,23 +169,29 @@ public class MemoryBytePatternSearcher {
 			throws IOException, CancelledException {
 
 		// if no restricted set, make restrict set the full block
-		AddressSet doneSet = new AddressSet(restrictSet);
+		AddressSet doneSet;
-		if (doneSet.isEmpty()) {
+		if (restrictSet == null || restrictSet.isEmpty()) {
-			doneSet.addRange(block.getStart(), block.getEnd());
+			doneSet = new AddressSet(block.getStart(), block.getEnd());
 		}
-		doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
+		else {
+			doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd());
+		}
+
+		long numInDoneSet = doneSet.getNumAddresses();
+		long numInBlock = block.getSize();
 
 		Address blockStartAddr = block.getStart();
 
 		// pull each range off the restricted set
+		long progress = monitor.getProgress();
 		AddressRangeIterator addressRanges = doneSet.getAddressRanges();
+		long numDone = 0;
 		while (addressRanges.hasNext()) {
 			monitor.checkCanceled();
-			AddressRange addressRange = addressRanges.next();
-
 			monitor.setMessage(searchName + " Search");
-			monitor.initialize(doneSet.getNumAddresses());
+			monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet)));
-			monitor.setProgress(0);
+			AddressRange addressRange = addressRanges.next();
+			long numAddressesInRange = addressRange.getLength();
 
 			ArrayList<Match> mymatches = new ArrayList<>();
 
@@ -155,25 +220,49 @@ public class MemoryBytePatternSearcher {
 			monitor.checkCanceled();
 
 			monitor.setMessage(searchName + " (Examine Matches)");
-			monitor.initialize(mymatches.size());
-			monitor.setProgress(0);
 
 			// TODO: DANGER there is much offset<-->address calculation here
 			//       should be OK, since they are all relative to the block.
+			long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet));
 			for (int i = 0; i < mymatches.size(); ++i) {
 				monitor.checkCanceled();
-				monitor.setProgress(i);
+				monitor.setProgress(
+					matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size())));
 				Match match = mymatches.get(i);
 				Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
 				if (!match.checkPostRules(streamoffset + blockOffset)) {
 					continue;
 				}
-				MatchAction[] matchactions = match.getMatchActions();
 
+				MatchAction[] matchactions = match.getMatchActions();
+				preMatchApply(matchactions, addr);
 				for (MatchAction matchaction : matchactions) {
 					matchaction.apply(program, addr, match);
 				}
+
+				postMatchApply(matchactions, addr);
 			}
+
+			numDone += numAddressesInRange;
 		}
 	}
+
+	/**
+	 * Called before any match rules are applied
+	 * @param matchactions actions that matched
+	 * @param addr address of match
+	 */
+	public void preMatchApply(MatchAction[] matchactions, Address addr) {
+		// override if any initialization needs to be done before rule application
+	}
+
+	/**
+	 * Called after any match rules are applied
+	 * Can use for cross post rule matching state application and cleanup.
+	 * @param matchactions actions that matched
+	 * @param addr adress of match
+	 */
+	public void postMatchApply(MatchAction[] matchactions, Address addr) {
+		// override if any cleanup from rule match application is needed
+	}
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java

@@ -37,6 +37,9 @@ public class Pattern extends DittedBitSequence {
 	private PostRule[] postrule;
 	private MatchAction[] actions;
 
+	/**
+	 * Construct an empty pattern.  Use XML to initialize
+	 */
 	public Pattern() {
 		markOffset = 0;
 		postrule = null;
@@ -44,6 +47,15 @@ public class Pattern extends DittedBitSequence {
 
 	}
 
+	/**
+	 * Construct the pattern based on a DittedByteSequence a match offset, post matching rules,
+	 * and a set of actions to take when the match occurs.
+	 * 
+	 * @param seq DittedByteSequence
+	 * @param offset offset from the actual match location to report a match
+	 * @param postArray post set of rules to check for the match
+	 * @param matchArray MatchActions to apply when a match occurs
+	 */
 	public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray,
 			MatchAction[] matchArray) {
 		super(seq);

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java

@@ -19,7 +19,18 @@ package ghidra.util.bytesearch;
  * Interface for factories that create Match Pattern classes
  */
 public interface PatternFactory {
+	/**
+	 * Get a named match action
+	 * 
+	 * @param nm name of action to find
+	 * @return match action with the given name, null otherwise
+	 */
 	public MatchAction getMatchActionByName(String nm);
 
+	/**
+	 * Get a named post match rule by name
+	 * @param nm name of the post rule
+	 * @return the post rule with the name, null otherwise
+	 */
 	public PostRule getPostRuleByName(String nm);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java

@@ -59,6 +59,9 @@ public class PatternPairSet {
 	private ArrayList<DittedBitSequence> preSequences;
 	private ArrayList<Pattern> postPatterns;
 
+	/**
+	 * Construct an empty PatternPairSet.  Use XML to initialize the pattern sets.
+	 */
 	public PatternPairSet() {
 		preSequences = new ArrayList<DittedBitSequence>();
 		postPatterns = new ArrayList<Pattern>();
@@ -84,12 +87,22 @@ public class PatternPairSet {
 		}
 	}
 
+	/**
+	 * Add this PatternPairSets post patterns to an existing arraylist of patterns.
+	 * @param postpats array to add this PatternPairSets post patterns into
+	 */
 	public void extractPostPatterns(ArrayList<Pattern> postpats) {
 		for (int i = 0; i < postPatterns.size(); ++i) {
 			postpats.add(postPatterns.get(i));
 		}
 	}
 
+	/**
+	 * Restore PatternPairSet from XML pull parser
+	 * @param parser XML pull parser
+	 * @param pfactory pattern factory user to construct patterns
+	 * @throws IOException if pull parsing fails
+	 */
 	public void restoreXml(XmlPullParser parser, PatternFactory pfactory) throws IOException {
 		XmlElement el = parser.start("patternpairs");
 		totalBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("totalbits"));

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java

@@ -21,7 +21,18 @@ import ghidra.xml.XmlPullParser;
  * Inteface for post match rules that are checked after a match is idenfied
  */
 public interface PostRule {
+	/**
+	 * Apply a post rule given the matching pattern and offset into the byte stream.
+	 * @param pat pattern that matched
+	 * @param matchoffset offset of the match
+	 * @return true if the PostRule is satisfied
+	 */
 	public boolean apply(Pattern pat, long matchoffset);
 
+	/**
+	 * Can restore state of instance PostRule from XML
+	 * 
+	 * @param parser XML pull parser
+	 */
 	public void restoreXml(XmlPullParser parser);
 }

Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java

@@ -27,19 +27,27 @@ import ghidra.util.task.TaskMonitor;
  */
 public class SequenceSearchState implements Comparable<SequenceSearchState> {
 
-	private static final int PATERN_ENDED = 10000000;
+	private static final int PATTERN_ENDED = Integer.MAX_VALUE;
 	private SequenceSearchState parent;
 	private ArrayList<DittedBitSequence> possible;		// Patterns that could still match in this state
 	private ArrayList<DittedBitSequence> success;		// Patterns that have matched successfully if we reached this state
 	private SequenceSearchState[] trans;				// State transitions based on next byte
 
-	public SequenceSearchState(SequenceSearchState par) {
+	/**
-		parent = par;
+	 * Construct a sub sequence state with a parent sequence
+	 * 
+	 * @param parent parent SequenceSearchState
+	 */
+	public SequenceSearchState(SequenceSearchState parent) {
+		this.parent = parent;
 		possible = new ArrayList<DittedBitSequence>();
 		success = null;
 		trans = null;
 	}
 
+	/**
+	 * @return maximum number of bytes that could be matched by this sequence
+	 */
 	public int getMaxSequenceSize() {
 		int max = 0;
 		for (DittedBitSequence element : possible) {
@@ -51,6 +59,12 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 		return max;
 	}
 
+	/**
+	 * Add a pattern to this search sequence.  The last pattern added is the successful
+	 * match pattern.
+	 * @param pat pattern to add
+	 * @param pos position within the current set of patterns to add this pattern
+	 */
 	public void addSequence(DittedBitSequence pat, int pos) {
 		possible.add(pat);
 		if (pos == pat.getSize()) {
@@ -61,6 +75,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 		}
 	}
 
+	/**
+	 * Sort the sequences that have been added 
+	 */
 	public void sortSequences() {
 		Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
 			@Override
@@ -150,9 +167,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 						}
 						i += 1;
 						j += 1;
-						thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
+						thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
 						oppat =
-							(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
+							(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
 					}
 					else if (thispat < oppat) {
 						if (curpat != thispat) {
@@ -160,7 +177,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 							curpat = thispat;
 						}
 						i += 1;
-						thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
+						thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
 					}
 					else {
 						if (curpat != oppat) {
@@ -169,7 +186,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 						}
 						j += 1;
 						oppat =
-							(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
+							(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
 					}
 				}
 				success = tmp;
@@ -177,6 +194,12 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 		}
 	}
 
+	/**
+	 * Try to match this Sequence to the byteArray, and add any matches to the match list
+	 * @param bytearray array of bytes to match
+	 * @param numbytes retrict number of bytes to allow to match
+	 * @param match list of matches, the result
+	 */
 	public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList<Match> match) {
 		int subindex = 0;
 		SequenceSearchState curstate = this;
@@ -242,6 +265,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 	 */
 	public void apply(InputStream in, long maxBytes, ArrayList<Match> match, TaskMonitor monitor)
 			throws IOException {
+		long progress = monitor.getProgress();
+
 		int maxSize = getMaxSequenceSize() + 1;
 		if (maxSize < 4096) {
 			maxSize = 4096;
@@ -317,7 +342,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 					if (monitor.isCancelled()) {
 						return;
 					}
-					monitor.setProgress(offset);
+					monitor.setProgress(progress + offset);
 				}
 				if (ra != secondBuf.length) {
 					fullBuffers = 1;
@@ -373,8 +398,15 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 		}
 	}
 
-	static public ArrayList<SequenceSearchState> buildTransitionLevel(
+	/**
-			ArrayList<SequenceSearchState> prev, int pos) {
+	 * Build a new transition level for the state machine
+	 * 
+	 * @param prev previous search sequences
+	 * @param pos position within the search sequence state for this level
+	 * @return list of possible new search states to be added to the state machine
+	 */
+	static ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,
+			int pos) {
 		ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
 		Iterator<SequenceSearchState> iterator = prev.iterator();
 		while (iterator.hasNext()) {			// For each current state
@@ -407,6 +439,11 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
 		return finalres;
 	}
 
+	/**
+	 * Build a search state machine from a list of DittedBitSequences
+	 * @param patterns bit sequence patterns
+	 * @return search state the will match the given sequences
+	 */
 	static public SequenceSearchState buildStateMachine(
 			ArrayList<? extends DittedBitSequence> patterns) {
 		SequenceSearchState root = new SequenceSearchState(null);

Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java

@@ -15,8 +15,6 @@
  */
 package ghidra.app.analyzers;
 
-import java.io.IOException;
-import java.io.InputStream;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -51,7 +49,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 	private static final String DESCRIPTION =
 		"Search for architecture specific byte patterns: typically starts of functions";
 	private static final String PRE_FUNCTION_MATCH_PROPERTY_NAME = "PreFunctionMatch";
-	private static final int RESTRICTED_PATTERN_BYTE_RANGE = 32;
 	private final static String OPTION_NAME_DATABLOCKS = "Search Data Blocks";
 	private static final String OPTION_DESCRIPTION_DATABLOCKS =
 		"Search for byte patterns in blocks that are not executable";
@@ -214,7 +211,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 			applyActionToSet(program, addr, funcResult, match);
 		}
 
-
 		protected boolean checkPreRequisites(Program program, Address addr) {
 			/**
 			 * If the match's mark point occurs in undefined data, schedule disassembly
@@ -254,10 +250,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 					return false;
 				}
 			}
-			
+
 			return true;
 		}
-		
 
 		protected void applyActionToSet(Program program, Address addr, AddressSet resultSet,
 				Match match) {
@@ -632,25 +627,34 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 			return false;
 		}
 
-		AddressSet restrictedSet = removeNotSearchedAddresses(program, set);
+		boolean doExecutableBlocksOnly = checkForExecuteBlock(program) && executableBlocksOnly;
 
 		// clear out any previous potential matches, because we are re-looking at these places
 		//   this will keep cruft from accumulating in the property map.
-		getOrCreatePotentialMatchPropertyMap(program).remove(restrictedSet);
+		getOrCreatePotentialMatchPropertyMap(program).remove(set);
 
-		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		MemoryBytePatternSearcher patternSearcher;
-		for (MemoryBlock block2 : blocks) {
+		patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) {
-			MemoryBlock block = block2;
+
-			if (!restrictedSet.intersects(block.getStart(), block.getEnd())) {
+			@Override
-				continue;
+			public void preMatchApply(MatchAction[] actions, Address addr) {
+				contextValueList = null; // make sure, only context from these actions used
 			}
-			try {
+
-				searchBlock(root, program, block, restrictedSet, monitor);
+			@Override
+			public void postMatchApply(MatchAction[] actions, Address addr) {
+				// Actions might have set context, check if postcondition failed first
+				if (!postreqFailedResult.contains(addr)) {
+					setCurrentContext(program, addr);
+				}
+				// get rid of the context list.
+				contextValueList = null;
 			}
-			catch (IOException e) {
+		};
-				log.appendMsg("Unable to scan block " + block.getName() + " for function starts");
+		patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly);
-			}
+
-		}
+		patternSearcher.search(program, set, monitor);
+
 		AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program);
 		if (!disassemResult.isEmpty()) {
 			analysisManager.disassemble(disassemResult);
@@ -708,26 +712,39 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 	}
 
 	/**
-	 * Get rid of any blocks from the address set that shouldn't be searched.
+	 * @return true - if there are any blocks marked executable
-	 *
-	 * @param program
-	 * @param bset
-	 * @return
 	 */
-	private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) {
+	private boolean checkForExecuteBlock(Program program) {
-		AddressSet restrictedSet = new AddressSet(bset);
 		MemoryBlock[] blocks = program.getMemory().getBlocks();
 		boolean hasExecutable = false;
+
 		for (MemoryBlock block : blocks) {
 			if (block.isExecute()) {
 				hasExecutable = true;
 			}
 		}
+		return hasExecutable;
+	}
+
+	/**
+	 * Get rid of any blocks from the address set that shouldn't be searched.
+	 *  Non-executable and non-initialized.
+	 *
+	 * @param program program
+	 * @param bset current set of restricted address ranges
+	 * @return return new set with blocks not to be searched removed
+	 */
+	private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) {
+		AddressSet restrictedSet = new AddressSet(bset);
+		MemoryBlock[] blocks = program.getMemory().getBlocks();
+		boolean hasExecutable = checkForExecuteBlock(program);
+
 		for (MemoryBlock block : blocks) {
 			if (!block.isInitialized()) {
 				restrictedSet.deleteRange(block.getStart(), block.getEnd());
 				continue;
 			}
+			// if 
 			if (executableBlocksOnly && hasExecutable) {
 				if (!block.isExecute()) {
 					restrictedSet.deleteRange(block.getStart(), block.getEnd());
@@ -816,94 +833,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
 		return patlist;
 	}
 
-	/**
-	 * Search through bytes of a memory block using the finite state machine -root-
-	 * Apply any additional rules for matching patterns.
-	 * @param program is the Program being searched
-	 * @param block is the specific block of bytes being searched
-	 * @throws IOException
-	 * @throws CancelledException
-	 */
-	private void searchBlock(SequenceSearchState root, Program program, MemoryBlock block,
-			AddressSetView restrictSet, TaskMonitor monitor)
-			throws IOException, CancelledException {
-
-		// if no restricted set, make restrict set the full block
-		AddressSet doneSet = new AddressSet(restrictSet);
-		if (doneSet.isEmpty()) {
-			doneSet.addRange(block.getStart(), block.getEnd());
-		}
-		doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
-
-		Address blockStartAddr = block.getStart();
-
-		// pull each range off the restricted set
-		AddressRangeIterator addressRanges = doneSet.getAddressRanges();
-		while (addressRanges.hasNext()) {
-			monitor.checkCanceled();
-			AddressRange addressRange = addressRanges.next();
-
-			monitor.setMessage("Function Search");
-			monitor.initialize(doneSet.getNumAddresses());
-			monitor.setProgress(0);
-
-			ArrayList<Match> mymatches = new ArrayList<>();
-
-			long streamoffset = blockStartAddr.getOffset();
-
-			// Give block a starting/ending point before this address to search
-			//    patterns might start before, since they have a pre-pattern
-			// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
-			//       normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
-			//       normally the whole memory blocks will get searched.
-			long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
-			blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
-			if (blockOffset <= 0) {
-				// don't go before the block start
-				blockOffset = 0;
-			}
-
-			// compute number of bytes in the range + 1, and don't search more than that.
-			long maxBlockSearchLength =
-				addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
-
-			InputStream data = block.getData();
-			data.skip(blockOffset);
-
-			root.apply(data, maxBlockSearchLength, mymatches, monitor);
-			monitor.checkCanceled();
-
-			monitor.setMessage("Function Search (Examine Matches)");
-			monitor.initialize(mymatches.size());
-			monitor.setProgress(0);
-
-			// TODO: DANGER there is much offset<-->address calculation here
-			//       should be OK, since they are all relative to the block.
-			for (int i = 0; i < mymatches.size(); ++i) {
-				monitor.checkCanceled();
-				monitor.setProgress(i);
-				Match match = mymatches.get(i);
-				Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
-				if (!match.checkPostRules(streamoffset + blockOffset)) {
-					continue;
-				}
-				MatchAction[] matchactions = match.getMatchActions();
-				contextValueList = null; // make sure, only context from these actions used
-				for (MatchAction matchaction : matchactions) {
-					matchaction.apply(program, addr, match);
-				}
-				// Actions might have set context, check if postcondition failed first
-				if (!postreqFailedResult.contains(addr)) {
-					setCurrentContext(program, addr);
-				}
-				else {
-					// didn't apply it, get rid of the context list.
-					contextValueList = null;
-				}
-			}
-		}
-	}
-
 	@Override
 	public MatchAction getMatchActionByName(String nm) {
 		if (nm.equals("funcstart")) {

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java

@@ -38,7 +38,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 		extends BackgroundCommand {
 
 	protected final String name;
-	protected final Address address;
+	private Address address;
 	protected final int count;
 	protected final DataValidationOptions validationOptions;
 	protected final DataApplyOptions applyOptions;
@@ -178,7 +178,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 			catch (InvalidInputException e) {
 				// Catch the exception and output the error, but still should create 
 				// associated data if possible, even though markup failed.
-				handleErrorMessage(program, name, getDataAddress(), getDataAddress(), e);
+				handleErrorMessage(program, name, address, address, e);
 				success = false;
 			}
 
@@ -191,7 +191,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 		}
 		catch (AddressOutOfBoundsException | CodeUnitInsertionException | DataTypeConflictException
 				| InvalidDataTypeException e) {
-			handleErrorMessage(program, name, getDataAddress(), getDataAddress(), e);
+			handleErrorMessage(program, name, address, address, e);
 			return false;
 		}
 	}
@@ -212,8 +212,8 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 			throw new CodeUnitInsertionException(
 				"Unable to get data type from model, " + model.getName() + ".");
 		}
-		if (!memory.getLoadedAndInitializedAddressSet().contains(getDataAddress())) {
+		if (!memory.getLoadedAndInitializedAddressSet().contains(address)) {
-			String message = "Can't create an " + dt.getName() + " @ " + getDataAddress() +
+			String message = "Can't create an " + dt.getName() + " @ " + address +
 				" which isn't in loaded and initialized memory for " + program.getName();
 			throw new CodeUnitInsertionException(message);
 		}
@@ -229,15 +229,14 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 		monitor.checkCanceled();
 
 		// Is the data type already applied at the address?
-		if (matchingDataExists(dt, program, getDataAddress())) {
+		if (matchingDataExists(dt, program, address)) {
 			return;
 		}
 
 		monitor.checkCanceled();
 
 		// Create data at the address using the datatype.
-		DataUtilities.createData(program, getDataAddress(), dt, dt.getLength(), false,
+		DataUtilities.createData(program, address, dt, dt.getLength(), false, getClearDataMode());
-			getClearDataMode());
 	}
 
 	/**
@@ -362,7 +361,17 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
 	 * 
 	 * @return the address of the data item being created.
 	 */
-	protected Address getDataAddress() {
+	final protected Address getDataAddress() {
 		return address;
 	}
+
+	/**
+	 * Set the address of the data item to be applied.
+	 * Can be used for sub classes that need to apply multiple data items.
+	 * 
+	 * @param addr set the current data address
+	 */
+	final protected void setDataAddress(Address addr) {
+		address = addr;
+	}
 }

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti4BackgroundCmd.java

@@ -42,8 +42,6 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 	private List<MemoryBlock> vfTableBlocks;
 	private List<Address> rtti4Locations;
 
-	private Address currentProcessingAddress;
-
 	/**
 	 * Constructs a command for applying an RTTI4 dataType at an address.
 	 * @param address the address where the data should be created using the data type.
@@ -79,14 +77,14 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 		List<Address> goodRtti4Locations = new ArrayList<Address>();
 		boolean succeeded = false;
 		for (Address addr : rtti4Locations) {
-			currentProcessingAddress = addr;
+			setDataAddress(addr);
 			succeeded |= super.doApplyTo(program, taskMonitor);
 			goodRtti4Locations.add(addr);
 		}
 
 		// if any succeeded and should create associated data, make the vftables all at one time
 		if (succeeded && applyOptions.shouldFollowData()) {
-			createaAssociatedVfTables(program, goodRtti4Locations, taskMonitor);
+			createAssociatedVfTables(program, goodRtti4Locations, taskMonitor);
 		}
 
 		return succeeded;
@@ -147,7 +145,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 		return cmd.applyTo(model.getProgram(), monitor);
 	}
 
-	private boolean createaAssociatedVfTables(Program program, List<Address> goodRtti4Locations,
+	private boolean createAssociatedVfTables(Program program, List<Address> goodRtti4Locations,
 			TaskMonitor taskMonitor) throws CancelledException {
 
 		MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables");
@@ -159,10 +157,6 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 			byte[] bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti4Address);
 
 			addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
-
-			bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti4Address);
-
-			addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
 		}
 
 		AddressSet searchSet = new AddressSet();
@@ -256,10 +250,9 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 
 			// Plate Comment
 			// Plate Comment
-			EHDataTypeUtilities.createPlateCommentIfNeeded(program,
+			EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
-				RttiUtil.CONST_PREFIX + RttiUtil.getDescriptorTypeNamespace(rtti0Model) +
+				RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER, RTTI_4_NAME,
-					Namespace.DELIMITER,
+				null, getDataAddress(), applyOptions);
-				RTTI_4_NAME, null, getDataAddress(), applyOptions);
 			monitor.checkCanceled();
 
 			// Label
@@ -271,15 +264,4 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
 
 		return true;
 	}
-
-	/**
-	 * Creating RTTI4 tables handles creating multiple tables at once.
-	 * The generic base class that handles getting the address will get the RTTI4 tabl address.
-	 * 
-	 * @return the current address to process
-	 */
-	@Override
-	protected Address getDataAddress() {
-		return currentProcessingAddress;
-	}
 }

Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java

@@ -240,12 +240,6 @@ public class RttiAnalyzer extends AbstractAnalyzer {
 				// 32-bit could have direct address in memory
 				bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti0Address);
 
-				addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
-					rtti0Address, bytes);
-
-				// or referenced by a shifted pointer based on program data organization shift amount
-				bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti0Address);
-
 				addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
 					rtti0Address, bytes);
 			}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/util/AddressSetPropertyMapDB.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +15,10 @@
  */
 package ghidra.program.database.util;
 
+import java.util.ConcurrentModificationException;
+
+import db.*;
+import db.util.ErrorHandler;
 import ghidra.program.database.ProgramDB;
 import ghidra.program.database.map.AddressMap;
 import ghidra.program.model.address.*;
@@ -26,11 +29,6 @@ import ghidra.util.exception.CancelledException;
 import ghidra.util.exception.DuplicateNameException;
 import ghidra.util.task.TaskMonitor;
 
-import java.util.ConcurrentModificationException;
-
-import db.*;
-import db.util.ErrorHandler;
-
 /**
  * AddressSetPropertyMap that uses a RangeMapDB to maintain a set of addresses.
  * 
@@ -74,8 +72,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 			DBHandle dbh = program.getDBHandle();
 			String tableName = AddressSetPropertyMapDB.TABLE_PREFIX + mapName;
 			if (dbh.getTable(tableName) != null) {
-				throw new DuplicateNameException("Address Set Property Map named " + mapName +
+				throw new DuplicateNameException(
-					" already exists.");
+					"Address Set Property Map named " + mapName + " already exists.");
 			}
 
 			return new AddressSetPropertyMapDB(program, mapName, program, addrMap, lock);
@@ -91,9 +89,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 		this.mapName = mapName;
 		this.lock = lock;
 
-		propertyMap =
+		propertyMap = new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
-			new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
+			program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
-				program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
 	}
 
 	@Override
@@ -114,7 +111,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#add(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void add(AddressSet addressSet) {
+	public void add(AddressSetView addressSet) {
 		checkDeleted();
 
 		lock.acquire();
@@ -135,7 +132,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#set(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void set(AddressSet addressSet) {
+	public void set(AddressSetView addressSet) {
 		checkDeleted();
 		lock.acquire();
 		try {
@@ -169,7 +166,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
 	 * @see ghidra.program.model.util.AddressSetPropertyMap#remove(ghidra.program.model.address.AddressSet)
 	 */
 	@Override
-	public void remove(AddressSet addressSet) {
+	public void remove(AddressSetView addressSet) {
 		checkDeleted();
 		lock.acquire();
 		try {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/util/AddressSetPropertyMap.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,59 +22,60 @@ import ghidra.program.model.address.*;
  *
  */
 public interface AddressSetPropertyMap {
-	
+
 	/**
 	 * Add the address range to the property map.
 	 * @param start start of the range
 	 * @param end end of the range
 	 */
 	void add(Address start, Address end);
-	
+
 	/**
 	 * Add the address set to the property map.
 	 * @param addressSet address set to add
 	 */
-	void add(AddressSet addressSet);
+	void add(AddressSetView addressSet);
-	
+
 	/**
 	 * Clear the property map and set it with the given address set. 
 	 * @param addressSet address set to use
 	 */
-	void set(AddressSet addressSet);
+	void set(AddressSetView addressSet);
+
 	/**
 	 * Remove the address range from the property map.
 	 * @param start start of the range
 	 * @param end end of the range
 	 */
 	void remove(Address start, Address end);
-	
+
 	/**
 	 * Remove the address set from the property map.
 	 * @param addressSet address set to remove
 	 */
-	void remove(AddressSet addressSet);
+	void remove(AddressSetView addressSet);
-	
+
 	/**
 	 * Return the address set for the property map.
 	 */
 	AddressSet getAddressSet();
-	
+
 	/**
 	 * Return an address iterator over the property map. 
 	 */
 	AddressIterator getAddresses();
-	
+
 	/**
 	 * Return an address range iterator over the property map.
 	 */
 	AddressRangeIterator getAddressRanges();
-	
+
 	/**
 	 * Clear the property map.
 	 *
 	 */
 	void clear();
-	
+
 	/**
 	 * Return whether the property map contains the given address.
 	 * @param addr address to check