From b51a9d7ff4955a5618dedb8d259ed4b76af43a2f Mon Sep 17 00:00:00 2001 From: emteere <47253321+emteere@users.noreply.github.com> Date: Fri, 14 Feb 2020 18:55:32 +0000 Subject: [PATCH] GT-3341_emteere_RTTIPerformance code review changes, refactored progress monitor --- .../ghidra/util/bytesearch/AlignRule.java | 46 ++--- .../GenericByteSequencePattern.java | 15 ++ .../util/bytesearch/GenericMatchAction.java | 5 +- .../java/ghidra/util/bytesearch/Match.java | 10 +- .../ghidra/util/bytesearch/MatchAction.java | 12 ++ .../bytesearch/MemoryBytePatternSearcher.java | 125 ++++++++++++-- .../java/ghidra/util/bytesearch/Pattern.java | 12 ++ .../util/bytesearch/PatternFactory.java | 11 ++ .../util/bytesearch/PatternPairSet.java | 13 ++ .../java/ghidra/util/bytesearch/PostRule.java | 11 ++ .../util/bytesearch/SequenceSearchState.java | 57 +++++-- .../app/analyzers/FunctionStartAnalyzer.java | 157 +++++------------- .../data/AbstractCreateDataBackgroundCmd.java | 27 ++- .../data/rtti/CreateRtti4BackgroundCmd.java | 30 +--- .../RttiAnalyzer.java | 6 - .../util/AddressSetPropertyMapDB.java | 25 ++- .../model/util/AddressSetPropertyMap.java | 26 +-- 17 files changed, 357 insertions(+), 231 deletions(-) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java index 314c57b43e..b6606476f8 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java @@ -20,20 +20,11 @@ import ghidra.xml.XmlElement; import ghidra.xml.XmlPullParser; /** - * ByteSearch post search rule when a pattern is found, The pattern must have a certain - * alignment at an offset from the location the pattern matches. The alignment is - * specified by the mask bits that must be zero. + * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain + * alignment at an offset from the location the pattern matches. * - * mark is the offset in bytes from the start of the matching pattern. - * - * align 2 = 0x1 - lower bit must be zero - * align 4 = 0x3 - lower two bits must be zero - * align 8 = 0x7 - lower three bits must be zero - * align 16 = 0xF - lower four bits must be zero - * .... - * Other strange alignments could be specified, but most likely the above suffice. - * - * The pattern can be constructed or restored from XML of the form: + * The pattern can be constructed or restored from XML of the form, + * where alignOffset=mark, alignmask=bits * * * @@ -41,27 +32,44 @@ import ghidra.xml.XmlPullParser; public class AlignRule implements PostRule { - private int mark; // Position, relative to start of pattern, to check alignment at - private int alignmask; // Mask of bits that must be zero + private int alignOffset; // Position, relative to start of pattern, to check alignment at + private int alignmask; // Mask of bits that must be zero public AlignRule() { } - public AlignRule(int mark, int alignmask) { - this.mark = mark; + /** + * ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain + * alignment at an offset from the location the pattern matches. The alignment is + * specified by the alignmask bits that must be zero. + * + * Normally alignOffset is 0, since most patterns will match at the address that must be aligned + * To align a match, use the following + * + * align to 2 = alignmask 0x1 - lower bit must be zero + * align to 4 = alignmask 0x3 - lower two bits must be zero + * align to 8 = alignmask 0x7 - lower three bits must be zero + * align to 16 = alignmask 0xF - lower four bits must be zero + * .... + * Other strange alignments could be specified, but most likely the above suffice. + * @param alignOffset - bytes offset from pattern to check for alignment + * @param alignmask - the mask where a 1 bit must be zero + */ + public AlignRule(int alignOffset, int alignmask) { + this.alignOffset = alignOffset; this.alignmask = alignmask; } @Override public boolean apply(Pattern pat, long matchoffset) { int off = (int) matchoffset; - return (((off + mark) & alignmask) == 0); + return (((off + alignOffset) & alignmask) == 0); } @Override public void restoreXml(XmlPullParser parser) { XmlElement el = parser.start("align"); - mark = SpecXmlUtils.decodeInt(el.getAttribute("mark")); + alignOffset = SpecXmlUtils.decodeInt(el.getAttribute("mark")); int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits")); alignmask = (1 << bits) - 1; parser.end(); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java index 773dd04208..be3dfa7533 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java @@ -29,6 +29,13 @@ import ghidra.program.model.data.DataType; public class GenericByteSequencePattern extends Pattern { + /** + * Construct a sequence of bytes with no mask, and associated action + * to be called if this pattern matches. + * + * @param bytesSequence sequence of bytes to match + * @param action action to apply if the match succeeds + */ public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction action) { super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]); @@ -36,6 +43,14 @@ public class GenericByteSequencePattern extends Pattern { matchActions[0] = action; } + /** + * Construct a sequence of bytes with a mask, and associated action + * to be called if this pattern matches. + * + * @param bytesSequence sequence of bytes to match + * @param mask mask, bits that are 1 must match the byteSequence bits + * @param action to apply if the match succeeds + */ public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask, GenericMatchAction action) { super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java index 7155e981fb..3b337c6fed 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java @@ -26,8 +26,9 @@ package ghidra.util.bytesearch; public class GenericMatchAction extends DummyMatchAction { T matchValue; - /* - * construct with an appropriate object that can be used when the action is applied + /** + * Construct a match action used when a match occurs for some GenericByteSequece + * @param matchValue specialized object used when match occurs */ public GenericMatchAction(T matchValue) { this.matchValue = matchValue; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java index 7da007cbe4..59dac95a56 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java @@ -24,9 +24,15 @@ package ghidra.util.bytesearch; * */ public class Match { - private DittedBitSequence sequence; // Pattern that matches - private long offset; // starting offset within bytestream of match + private DittedBitSequence sequence; // Pattern that matched + private long offset; // Offset within bytestream where the match occurred + /** + * Construct a Match of a DittedBitSequence at an offset within a byte stream. + * Object normally used when a match occurs during a MemoryBytePatternSearch. + * @param sequence that matched + * @param offset from the start of byte stream where the matched occured + */ public Match(DittedBitSequence sequence, long offset) { this.sequence = sequence; this.offset = offset; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java index 750aedeb0f..6c81bfe9a9 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java @@ -23,7 +23,19 @@ import ghidra.xml.XmlPullParser; * Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern */ public interface MatchAction { + /** + * Apply the match action to the program at the address. + * + * @param program program in which the match occurred + * @param addr where the match occured + * @param match information about the match that occurred + */ public void apply(Program program, Address addr, Match match); + /** + * Action can be constructed from XML + * + * @param parser XML pull parser to restore action from XML + */ public void restoreXml(XmlPullParser parser); } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java index 6683156057..d65bece053 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java @@ -28,6 +28,7 @@ import ghidra.util.task.TaskMonitor; /** * Multi pattern/mask/action memory searcher + * Patterns must be supplied/added, or a pre-initialized searchState supplied * * Preload search patterns and actions, then call search method. */ @@ -35,13 +36,20 @@ import ghidra.util.task.TaskMonitor; public class MemoryBytePatternSearcher { private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32; + SequenceSearchState root = null; + ArrayList patternList; private String searchName = ""; + private boolean doExecutableBlocksOnly = false; // only search executable blocks + + private long numToSearch = 0; + private long numSearched = 0; + /** * Create with pre-created patternList - * + * @param searchName name of search * @param patternList - list of patterns(bytes/mask/action) */ public MemoryBytePatternSearcher(String searchName, ArrayList patternList) { @@ -49,8 +57,19 @@ public class MemoryBytePatternSearcher { this.patternList = new ArrayList(patternList); } + /** + * Create with an initialized SequenceSearchState + * @param searchName name of search + * @param root search state pre-initialized + */ + public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) { + this.searchName = searchName; + this.root = root; + } + /** * Create with no patternList, must add patterns before searching + * @param searchName name of search * */ public MemoryBytePatternSearcher(String searchName) { @@ -66,6 +85,10 @@ public class MemoryBytePatternSearcher { patternList.add(pattern); } + public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) { + this.doExecutableBlocksOnly = doExecutableBlocksOnly; + } + /** * Search initialized memory blocks for all patterns(bytes/mask/action). * Call associated action for each pattern matched. @@ -78,23 +101,59 @@ public class MemoryBytePatternSearcher { */ public void search(Program program, AddressSetView searchSet, TaskMonitor monitor) throws CancelledException { - SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList); + if (root == null) { + root = SequenceSearchState.buildStateMachine(patternList); + } + + numToSearch = getNumToSearch(program, searchSet); + monitor.setMessage(searchName + " Search"); + monitor.initialize(numToSearch); MemoryBlock[] blocks = program.getMemory().getBlocks(); - for (MemoryBlock block2 : blocks) { - MemoryBlock block = block2; - if (!searchSet.intersects(block.getStart(), block.getEnd())) { + for (MemoryBlock block : blocks) { + monitor.setProgress(numSearched); + // check if entire block has anything that is searchable + if (!block.isInitialized()) { continue; } + if (doExecutableBlocksOnly && !block.isExecute()) { + continue; + } + if (searchSet != null && !searchSet.isEmpty() && + !searchSet.intersects(block.getStart(), block.getEnd())) { + continue; + } + try { searchBlock(root, program, block, searchSet, monitor); } catch (IOException e) { - Msg.error(this, "Unable to scan block " + block.getName() + " for patterns"); + Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName); } + numSearched += block.getSize(); } } + private long getNumToSearch(Program program, AddressSetView searchSet) { + long numAddresses = 0; + MemoryBlock[] blocks = program.getMemory().getBlocks(); + for (MemoryBlock block : blocks) { + // check if entire block has anything that is searchable + if (!block.isInitialized()) { + continue; + } + if (doExecutableBlocksOnly && !block.isExecute()) { + continue; + } + if (searchSet != null && !searchSet.isEmpty() && + !searchSet.intersects(block.getStart(), block.getEnd())) { + continue; + } + numAddresses += block.getSize(); + } + return numAddresses; + } + /** * Search through bytes of a memory block using the finite state machine -root- * Apply any additional rules for matching patterns. @@ -110,23 +169,29 @@ public class MemoryBytePatternSearcher { throws IOException, CancelledException { // if no restricted set, make restrict set the full block - AddressSet doneSet = new AddressSet(restrictSet); - if (doneSet.isEmpty()) { - doneSet.addRange(block.getStart(), block.getEnd()); + AddressSet doneSet; + if (restrictSet == null || restrictSet.isEmpty()) { + doneSet = new AddressSet(block.getStart(), block.getEnd()); } - doneSet = doneSet.intersectRange(block.getStart(), block.getEnd()); + else { + doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd()); + } + + long numInDoneSet = doneSet.getNumAddresses(); + long numInBlock = block.getSize(); Address blockStartAddr = block.getStart(); // pull each range off the restricted set + long progress = monitor.getProgress(); AddressRangeIterator addressRanges = doneSet.getAddressRanges(); + long numDone = 0; while (addressRanges.hasNext()) { monitor.checkCanceled(); - AddressRange addressRange = addressRanges.next(); - monitor.setMessage(searchName + " Search"); - monitor.initialize(doneSet.getNumAddresses()); - monitor.setProgress(0); + monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet))); + AddressRange addressRange = addressRanges.next(); + long numAddressesInRange = addressRange.getLength(); ArrayList mymatches = new ArrayList<>(); @@ -155,25 +220,49 @@ public class MemoryBytePatternSearcher { monitor.checkCanceled(); monitor.setMessage(searchName + " (Examine Matches)"); - monitor.initialize(mymatches.size()); - monitor.setProgress(0); // TODO: DANGER there is much offset<-->address calculation here // should be OK, since they are all relative to the block. + long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet)); for (int i = 0; i < mymatches.size(); ++i) { monitor.checkCanceled(); - monitor.setProgress(i); + monitor.setProgress( + matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size()))); Match match = mymatches.get(i); Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset); if (!match.checkPostRules(streamoffset + blockOffset)) { continue; } - MatchAction[] matchactions = match.getMatchActions(); + MatchAction[] matchactions = match.getMatchActions(); + preMatchApply(matchactions, addr); for (MatchAction matchaction : matchactions) { matchaction.apply(program, addr, match); } + + postMatchApply(matchactions, addr); } + + numDone += numAddressesInRange; } } + + /** + * Called before any match rules are applied + * @param matchactions actions that matched + * @param addr address of match + */ + public void preMatchApply(MatchAction[] matchactions, Address addr) { + // override if any initialization needs to be done before rule application + } + + /** + * Called after any match rules are applied + * Can use for cross post rule matching state application and cleanup. + * @param matchactions actions that matched + * @param addr adress of match + */ + public void postMatchApply(MatchAction[] matchactions, Address addr) { + // override if any cleanup from rule match application is needed + } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java index 1b7dc035a1..9ba00c0662 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java @@ -37,6 +37,9 @@ public class Pattern extends DittedBitSequence { private PostRule[] postrule; private MatchAction[] actions; + /** + * Construct an empty pattern. Use XML to initialize + */ public Pattern() { markOffset = 0; postrule = null; @@ -44,6 +47,15 @@ public class Pattern extends DittedBitSequence { } + /** + * Construct the pattern based on a DittedByteSequence a match offset, post matching rules, + * and a set of actions to take when the match occurs. + * + * @param seq DittedByteSequence + * @param offset offset from the actual match location to report a match + * @param postArray post set of rules to check for the match + * @param matchArray MatchActions to apply when a match occurs + */ public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, MatchAction[] matchArray) { super(seq); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java index 47b29373da..cfbe60acd7 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java @@ -19,7 +19,18 @@ package ghidra.util.bytesearch; * Interface for factories that create Match Pattern classes */ public interface PatternFactory { + /** + * Get a named match action + * + * @param nm name of action to find + * @return match action with the given name, null otherwise + */ public MatchAction getMatchActionByName(String nm); + /** + * Get a named post match rule by name + * @param nm name of the post rule + * @return the post rule with the name, null otherwise + */ public PostRule getPostRuleByName(String nm); } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java index 8d621220ef..bd6fe348c5 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java @@ -59,6 +59,9 @@ public class PatternPairSet { private ArrayList preSequences; private ArrayList postPatterns; + /** + * Construct an empty PatternPairSet. Use XML to initialize the pattern sets. + */ public PatternPairSet() { preSequences = new ArrayList(); postPatterns = new ArrayList(); @@ -84,12 +87,22 @@ public class PatternPairSet { } } + /** + * Add this PatternPairSets post patterns to an existing arraylist of patterns. + * @param postpats array to add this PatternPairSets post patterns into + */ public void extractPostPatterns(ArrayList postpats) { for (int i = 0; i < postPatterns.size(); ++i) { postpats.add(postPatterns.get(i)); } } + /** + * Restore PatternPairSet from XML pull parser + * @param parser XML pull parser + * @param pfactory pattern factory user to construct patterns + * @throws IOException if pull parsing fails + */ public void restoreXml(XmlPullParser parser, PatternFactory pfactory) throws IOException { XmlElement el = parser.start("patternpairs"); totalBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("totalbits")); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java index 71e6255761..0f6c6a20df 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java @@ -21,7 +21,18 @@ import ghidra.xml.XmlPullParser; * Inteface for post match rules that are checked after a match is idenfied */ public interface PostRule { + /** + * Apply a post rule given the matching pattern and offset into the byte stream. + * @param pat pattern that matched + * @param matchoffset offset of the match + * @return true if the PostRule is satisfied + */ public boolean apply(Pattern pat, long matchoffset); + /** + * Can restore state of instance PostRule from XML + * + * @param parser XML pull parser + */ public void restoreXml(XmlPullParser parser); } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java index fed499b558..fe32063619 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java @@ -27,19 +27,27 @@ import ghidra.util.task.TaskMonitor; */ public class SequenceSearchState implements Comparable { - private static final int PATERN_ENDED = 10000000; + private static final int PATTERN_ENDED = Integer.MAX_VALUE; private SequenceSearchState parent; private ArrayList possible; // Patterns that could still match in this state private ArrayList success; // Patterns that have matched successfully if we reached this state private SequenceSearchState[] trans; // State transitions based on next byte - public SequenceSearchState(SequenceSearchState par) { - parent = par; + /** + * Construct a sub sequence state with a parent sequence + * + * @param parent parent SequenceSearchState + */ + public SequenceSearchState(SequenceSearchState parent) { + this.parent = parent; possible = new ArrayList(); success = null; trans = null; } + /** + * @return maximum number of bytes that could be matched by this sequence + */ public int getMaxSequenceSize() { int max = 0; for (DittedBitSequence element : possible) { @@ -51,6 +59,12 @@ public class SequenceSearchState implements Comparable { return max; } + /** + * Add a pattern to this search sequence. The last pattern added is the successful + * match pattern. + * @param pat pattern to add + * @param pos position within the current set of patterns to add this pattern + */ public void addSequence(DittedBitSequence pat, int pos) { possible.add(pat); if (pos == pat.getSize()) { @@ -61,6 +75,9 @@ public class SequenceSearchState implements Comparable { } } + /** + * Sort the sequences that have been added + */ public void sortSequences() { Comparator comp = new Comparator() { @Override @@ -150,9 +167,9 @@ public class SequenceSearchState implements Comparable { } i += 1; j += 1; - thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex(); + thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex(); oppat = - (j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex(); + (j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex(); } else if (thispat < oppat) { if (curpat != thispat) { @@ -160,7 +177,7 @@ public class SequenceSearchState implements Comparable { curpat = thispat; } i += 1; - thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex(); + thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex(); } else { if (curpat != oppat) { @@ -169,7 +186,7 @@ public class SequenceSearchState implements Comparable { } j += 1; oppat = - (j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex(); + (j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex(); } } success = tmp; @@ -177,6 +194,12 @@ public class SequenceSearchState implements Comparable { } } + /** + * Try to match this Sequence to the byteArray, and add any matches to the match list + * @param bytearray array of bytes to match + * @param numbytes retrict number of bytes to allow to match + * @param match list of matches, the result + */ public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList match) { int subindex = 0; SequenceSearchState curstate = this; @@ -242,6 +265,8 @@ public class SequenceSearchState implements Comparable { */ public void apply(InputStream in, long maxBytes, ArrayList match, TaskMonitor monitor) throws IOException { + long progress = monitor.getProgress(); + int maxSize = getMaxSequenceSize() + 1; if (maxSize < 4096) { maxSize = 4096; @@ -317,7 +342,7 @@ public class SequenceSearchState implements Comparable { if (monitor.isCancelled()) { return; } - monitor.setProgress(offset); + monitor.setProgress(progress + offset); } if (ra != secondBuf.length) { fullBuffers = 1; @@ -373,8 +398,15 @@ public class SequenceSearchState implements Comparable { } } - static public ArrayList buildTransitionLevel( - ArrayList prev, int pos) { + /** + * Build a new transition level for the state machine + * + * @param prev previous search sequences + * @param pos position within the search sequence state for this level + * @return list of possible new search states to be added to the state machine + */ + static ArrayList buildTransitionLevel(ArrayList prev, + int pos) { ArrayList res = new ArrayList(); Iterator iterator = prev.iterator(); while (iterator.hasNext()) { // For each current state @@ -407,6 +439,11 @@ public class SequenceSearchState implements Comparable { return finalres; } + /** + * Build a search state machine from a list of DittedBitSequences + * @param patterns bit sequence patterns + * @return search state the will match the given sequences + */ static public SequenceSearchState buildStateMachine( ArrayList patterns) { SequenceSearchState root = new SequenceSearchState(null); diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java index 24e5061152..908293450b 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java +++ b/Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java @@ -15,8 +15,6 @@ */ package ghidra.app.analyzers; -import java.io.IOException; -import java.io.InputStream; import java.math.BigInteger; import java.util.ArrayList; import java.util.Iterator; @@ -51,7 +49,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa private static final String DESCRIPTION = "Search for architecture specific byte patterns: typically starts of functions"; private static final String PRE_FUNCTION_MATCH_PROPERTY_NAME = "PreFunctionMatch"; - private static final int RESTRICTED_PATTERN_BYTE_RANGE = 32; private final static String OPTION_NAME_DATABLOCKS = "Search Data Blocks"; private static final String OPTION_DESCRIPTION_DATABLOCKS = "Search for byte patterns in blocks that are not executable"; @@ -214,7 +211,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa applyActionToSet(program, addr, funcResult, match); } - protected boolean checkPreRequisites(Program program, Address addr) { /** * If the match's mark point occurs in undefined data, schedule disassembly @@ -254,10 +250,9 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa return false; } } - + return true; } - protected void applyActionToSet(Program program, Address addr, AddressSet resultSet, Match match) { @@ -632,25 +627,34 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa return false; } - AddressSet restrictedSet = removeNotSearchedAddresses(program, set); + boolean doExecutableBlocksOnly = checkForExecuteBlock(program) && executableBlocksOnly; // clear out any previous potential matches, because we are re-looking at these places // this will keep cruft from accumulating in the property map. - getOrCreatePotentialMatchPropertyMap(program).remove(restrictedSet); + getOrCreatePotentialMatchPropertyMap(program).remove(set); - MemoryBlock[] blocks = program.getMemory().getBlocks(); - for (MemoryBlock block2 : blocks) { - MemoryBlock block = block2; - if (!restrictedSet.intersects(block.getStart(), block.getEnd())) { - continue; + MemoryBytePatternSearcher patternSearcher; + patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) { + + @Override + public void preMatchApply(MatchAction[] actions, Address addr) { + contextValueList = null; // make sure, only context from these actions used } - try { - searchBlock(root, program, block, restrictedSet, monitor); + + @Override + public void postMatchApply(MatchAction[] actions, Address addr) { + // Actions might have set context, check if postcondition failed first + if (!postreqFailedResult.contains(addr)) { + setCurrentContext(program, addr); + } + // get rid of the context list. + contextValueList = null; } - catch (IOException e) { - log.appendMsg("Unable to scan block " + block.getName() + " for function starts"); - } - } + }; + patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly); + + patternSearcher.search(program, set, monitor); + AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program); if (!disassemResult.isEmpty()) { analysisManager.disassemble(disassemResult); @@ -708,26 +712,39 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa } /** - * Get rid of any blocks from the address set that shouldn't be searched. - * - * @param program - * @param bset - * @return + * @return true - if there are any blocks marked executable */ - private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) { - AddressSet restrictedSet = new AddressSet(bset); + private boolean checkForExecuteBlock(Program program) { MemoryBlock[] blocks = program.getMemory().getBlocks(); boolean hasExecutable = false; + for (MemoryBlock block : blocks) { if (block.isExecute()) { hasExecutable = true; } } + return hasExecutable; + } + + /** + * Get rid of any blocks from the address set that shouldn't be searched. + * Non-executable and non-initialized. + * + * @param program program + * @param bset current set of restricted address ranges + * @return return new set with blocks not to be searched removed + */ + private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) { + AddressSet restrictedSet = new AddressSet(bset); + MemoryBlock[] blocks = program.getMemory().getBlocks(); + boolean hasExecutable = checkForExecuteBlock(program); + for (MemoryBlock block : blocks) { if (!block.isInitialized()) { restrictedSet.deleteRange(block.getStart(), block.getEnd()); continue; } + // if if (executableBlocksOnly && hasExecutable) { if (!block.isExecute()) { restrictedSet.deleteRange(block.getStart(), block.getEnd()); @@ -816,94 +833,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa return patlist; } - /** - * Search through bytes of a memory block using the finite state machine -root- - * Apply any additional rules for matching patterns. - * @param program is the Program being searched - * @param block is the specific block of bytes being searched - * @throws IOException - * @throws CancelledException - */ - private void searchBlock(SequenceSearchState root, Program program, MemoryBlock block, - AddressSetView restrictSet, TaskMonitor monitor) - throws IOException, CancelledException { - - // if no restricted set, make restrict set the full block - AddressSet doneSet = new AddressSet(restrictSet); - if (doneSet.isEmpty()) { - doneSet.addRange(block.getStart(), block.getEnd()); - } - doneSet = doneSet.intersectRange(block.getStart(), block.getEnd()); - - Address blockStartAddr = block.getStart(); - - // pull each range off the restricted set - AddressRangeIterator addressRanges = doneSet.getAddressRanges(); - while (addressRanges.hasNext()) { - monitor.checkCanceled(); - AddressRange addressRange = addressRanges.next(); - - monitor.setMessage("Function Search"); - monitor.initialize(doneSet.getNumAddresses()); - monitor.setProgress(0); - - ArrayList mymatches = new ArrayList<>(); - - long streamoffset = blockStartAddr.getOffset(); - - // Give block a starting/ending point before this address to search - // patterns might start before, since they have a pre-pattern - // TODO: this is dangerous, since pattern might be very big, but the set should be restricted - // normally only when we are searching for more matching patterns that had a postrule that didn't satisfy - // normally the whole memory blocks will get searched. - long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr); - blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE; - if (blockOffset <= 0) { - // don't go before the block start - blockOffset = 0; - } - - // compute number of bytes in the range + 1, and don't search more than that. - long maxBlockSearchLength = - addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1; - - InputStream data = block.getData(); - data.skip(blockOffset); - - root.apply(data, maxBlockSearchLength, mymatches, monitor); - monitor.checkCanceled(); - - monitor.setMessage("Function Search (Examine Matches)"); - monitor.initialize(mymatches.size()); - monitor.setProgress(0); - - // TODO: DANGER there is much offset<-->address calculation here - // should be OK, since they are all relative to the block. - for (int i = 0; i < mymatches.size(); ++i) { - monitor.checkCanceled(); - monitor.setProgress(i); - Match match = mymatches.get(i); - Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset); - if (!match.checkPostRules(streamoffset + blockOffset)) { - continue; - } - MatchAction[] matchactions = match.getMatchActions(); - contextValueList = null; // make sure, only context from these actions used - for (MatchAction matchaction : matchactions) { - matchaction.apply(program, addr, match); - } - // Actions might have set context, check if postcondition failed first - if (!postreqFailedResult.contains(addr)) { - setCurrentContext(program, addr); - } - else { - // didn't apply it, get rid of the context list. - contextValueList = null; - } - } - } - } - @Override public MatchAction getMatchActionByName(String nm) { if (nm.equals("funcstart")) { diff --git a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java index 631fce5623..76c3c1a213 100644 --- a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java +++ b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java @@ -38,7 +38,7 @@ public abstract class AbstractCreateDataBackgroundCmd vfTableBlocks; private List
rtti4Locations; - private Address currentProcessingAddress; - /** * Constructs a command for applying an RTTI4 dataType at an address. * @param address the address where the data should be created using the data type. @@ -79,14 +77,14 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd goodRtti4Locations = new ArrayList
(); boolean succeeded = false; for (Address addr : rtti4Locations) { - currentProcessingAddress = addr; + setDataAddress(addr); succeeded |= super.doApplyTo(program, taskMonitor); goodRtti4Locations.add(addr); } // if any succeeded and should create associated data, make the vftables all at one time if (succeeded && applyOptions.shouldFollowData()) { - createaAssociatedVfTables(program, goodRtti4Locations, taskMonitor); + createAssociatedVfTables(program, goodRtti4Locations, taskMonitor); } return succeeded; @@ -147,7 +145,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd goodRtti4Locations, + private boolean createAssociatedVfTables(Program program, List
goodRtti4Locations, TaskMonitor taskMonitor) throws CancelledException { MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables"); @@ -159,10 +157,6 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd