GT-3341_emteere_RTTIPerformance code review changes, refactored progress

monitor
This commit is contained in:
emteere 2020-02-14 18:55:32 +00:00
parent b426f065f5
commit b51a9d7ff4
17 changed files with 357 additions and 231 deletions

View file

@ -20,20 +20,11 @@ import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* ByteSearch post search rule when a pattern is found, The pattern must have a certain
* alignment at an offset from the location the pattern matches. The alignment is
* specified by the mask bits that must be zero.
* ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
* alignment at an offset from the location the pattern matches.
*
* mark is the offset in bytes from the start of the matching pattern.
*
* align 2 = 0x1 - lower bit must be zero
* align 4 = 0x3 - lower two bits must be zero
* align 8 = 0x7 - lower three bits must be zero
* align 16 = 0xF - lower four bits must be zero
* ....
* Other strange alignments could be specified, but most likely the above suffice.
*
* The pattern can be constructed or restored from XML of the form:
* The pattern can be constructed or restored from XML of the form,
* where alignOffset=mark, alignmask=bits
*
* <align mark="0" bits="1"/>
*
@ -41,27 +32,44 @@ import ghidra.xml.XmlPullParser;
public class AlignRule implements PostRule {
private int mark; // Position, relative to start of pattern, to check alignment at
private int alignmask; // Mask of bits that must be zero
private int alignOffset; // Position, relative to start of pattern, to check alignment at
private int alignmask; // Mask of bits that must be zero
public AlignRule() {
}
public AlignRule(int mark, int alignmask) {
this.mark = mark;
/**
* ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
* alignment at an offset from the location the pattern matches. The alignment is
* specified by the alignmask bits that must be zero.
*
* Normally alignOffset is 0, since most patterns will match at the address that must be aligned
* To align a match, use the following
*
* align to 2 = alignmask 0x1 - lower bit must be zero
* align to 4 = alignmask 0x3 - lower two bits must be zero
* align to 8 = alignmask 0x7 - lower three bits must be zero
* align to 16 = alignmask 0xF - lower four bits must be zero
* ....
* Other strange alignments could be specified, but most likely the above suffice.
* @param alignOffset - bytes offset from pattern to check for alignment
* @param alignmask - the mask where a 1 bit must be zero
*/
public AlignRule(int alignOffset, int alignmask) {
this.alignOffset = alignOffset;
this.alignmask = alignmask;
}
@Override
public boolean apply(Pattern pat, long matchoffset) {
int off = (int) matchoffset;
return (((off + mark) & alignmask) == 0);
return (((off + alignOffset) & alignmask) == 0);
}
@Override
public void restoreXml(XmlPullParser parser) {
XmlElement el = parser.start("align");
mark = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
alignOffset = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
alignmask = (1 << bits) - 1;
parser.end();

View file

@ -29,6 +29,13 @@ import ghidra.program.model.data.DataType;
public class GenericByteSequencePattern<T> extends Pattern {
/**
* Construct a sequence of bytes with no mask, and associated action
* to be called if this pattern matches.
*
* @param bytesSequence sequence of bytes to match
* @param action action to apply if the match succeeds
*/
public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction<T> action) {
super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]);
@ -36,6 +43,14 @@ public class GenericByteSequencePattern<T> extends Pattern {
matchActions[0] = action;
}
/**
* Construct a sequence of bytes with a mask, and associated action
* to be called if this pattern matches.
*
* @param bytesSequence sequence of bytes to match
* @param mask mask, bits that are 1 must match the byteSequence bits
* @param action to apply if the match succeeds
*/
public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask,
GenericMatchAction<DataType> action) {
super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]);

View file

@ -26,8 +26,9 @@ package ghidra.util.bytesearch;
public class GenericMatchAction<T> extends DummyMatchAction {
T matchValue;
/*
* construct with an appropriate object that can be used when the action is applied
/**
* Construct a match action used when a match occurs for some GenericByteSequece
* @param matchValue specialized object used when match occurs
*/
public GenericMatchAction(T matchValue) {
this.matchValue = matchValue;

View file

@ -24,9 +24,15 @@ package ghidra.util.bytesearch;
*
*/
public class Match {
private DittedBitSequence sequence; // Pattern that matches
private long offset; // starting offset within bytestream of match
private DittedBitSequence sequence; // Pattern that matched
private long offset; // Offset within bytestream where the match occurred
/**
* Construct a Match of a DittedBitSequence at an offset within a byte stream.
* Object normally used when a match occurs during a MemoryBytePatternSearch.
* @param sequence that matched
* @param offset from the start of byte stream where the matched occured
*/
public Match(DittedBitSequence sequence, long offset) {
this.sequence = sequence;
this.offset = offset;

View file

@ -23,7 +23,19 @@ import ghidra.xml.XmlPullParser;
* Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern
*/
public interface MatchAction {
/**
* Apply the match action to the program at the address.
*
* @param program program in which the match occurred
* @param addr where the match occured
* @param match information about the match that occurred
*/
public void apply(Program program, Address addr, Match match);
/**
* Action can be constructed from XML
*
* @param parser XML pull parser to restore action from XML
*/
public void restoreXml(XmlPullParser parser);
}

View file

@ -28,6 +28,7 @@ import ghidra.util.task.TaskMonitor;
/**
* Multi pattern/mask/action memory searcher
* Patterns must be supplied/added, or a pre-initialized searchState supplied
*
* Preload search patterns and actions, then call search method.
*/
@ -35,13 +36,20 @@ import ghidra.util.task.TaskMonitor;
public class MemoryBytePatternSearcher {
private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;
SequenceSearchState root = null;
ArrayList<Pattern> patternList;
private String searchName = "";
private boolean doExecutableBlocksOnly = false; // only search executable blocks
private long numToSearch = 0;
private long numSearched = 0;
/**
* Create with pre-created patternList
*
* @param searchName name of search
* @param patternList - list of patterns(bytes/mask/action)
*/
public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {
@ -49,8 +57,19 @@ public class MemoryBytePatternSearcher {
this.patternList = new ArrayList<Pattern>(patternList);
}
/**
* Create with an initialized SequenceSearchState
* @param searchName name of search
* @param root search state pre-initialized
*/
public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) {
this.searchName = searchName;
this.root = root;
}
/**
* Create with no patternList, must add patterns before searching
* @param searchName name of search
*
*/
public MemoryBytePatternSearcher(String searchName) {
@ -66,6 +85,10 @@ public class MemoryBytePatternSearcher {
patternList.add(pattern);
}
public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) {
this.doExecutableBlocksOnly = doExecutableBlocksOnly;
}
/**
* Search initialized memory blocks for all patterns(bytes/mask/action).
* Call associated action for each pattern matched.
@ -78,23 +101,59 @@ public class MemoryBytePatternSearcher {
*/
public void search(Program program, AddressSetView searchSet, TaskMonitor monitor)
throws CancelledException {
SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList);
if (root == null) {
root = SequenceSearchState.buildStateMachine(patternList);
}
numToSearch = getNumToSearch(program, searchSet);
monitor.setMessage(searchName + " Search");
monitor.initialize(numToSearch);
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block2 : blocks) {
MemoryBlock block = block2;
if (!searchSet.intersects(block.getStart(), block.getEnd())) {
for (MemoryBlock block : blocks) {
monitor.setProgress(numSearched);
// check if entire block has anything that is searchable
if (!block.isInitialized()) {
continue;
}
if (doExecutableBlocksOnly && !block.isExecute()) {
continue;
}
if (searchSet != null && !searchSet.isEmpty() &&
!searchSet.intersects(block.getStart(), block.getEnd())) {
continue;
}
try {
searchBlock(root, program, block, searchSet, monitor);
}
catch (IOException e) {
Msg.error(this, "Unable to scan block " + block.getName() + " for patterns");
Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName);
}
numSearched += block.getSize();
}
}
private long getNumToSearch(Program program, AddressSetView searchSet) {
long numAddresses = 0;
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block : blocks) {
// check if entire block has anything that is searchable
if (!block.isInitialized()) {
continue;
}
if (doExecutableBlocksOnly && !block.isExecute()) {
continue;
}
if (searchSet != null && !searchSet.isEmpty() &&
!searchSet.intersects(block.getStart(), block.getEnd())) {
continue;
}
numAddresses += block.getSize();
}
return numAddresses;
}
/**
* Search through bytes of a memory block using the finite state machine -root-
* Apply any additional rules for matching patterns.
@ -110,23 +169,29 @@ public class MemoryBytePatternSearcher {
throws IOException, CancelledException {
// if no restricted set, make restrict set the full block
AddressSet doneSet = new AddressSet(restrictSet);
if (doneSet.isEmpty()) {
doneSet.addRange(block.getStart(), block.getEnd());
AddressSet doneSet;
if (restrictSet == null || restrictSet.isEmpty()) {
doneSet = new AddressSet(block.getStart(), block.getEnd());
}
doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
else {
doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd());
}
long numInDoneSet = doneSet.getNumAddresses();
long numInBlock = block.getSize();
Address blockStartAddr = block.getStart();
// pull each range off the restricted set
long progress = monitor.getProgress();
AddressRangeIterator addressRanges = doneSet.getAddressRanges();
long numDone = 0;
while (addressRanges.hasNext()) {
monitor.checkCanceled();
AddressRange addressRange = addressRanges.next();
monitor.setMessage(searchName + " Search");
monitor.initialize(doneSet.getNumAddresses());
monitor.setProgress(0);
monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet)));
AddressRange addressRange = addressRanges.next();
long numAddressesInRange = addressRange.getLength();
ArrayList<Match> mymatches = new ArrayList<>();
@ -155,25 +220,49 @@ public class MemoryBytePatternSearcher {
monitor.checkCanceled();
monitor.setMessage(searchName + " (Examine Matches)");
monitor.initialize(mymatches.size());
monitor.setProgress(0);
// TODO: DANGER there is much offset<-->address calculation here
// should be OK, since they are all relative to the block.
long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet));
for (int i = 0; i < mymatches.size(); ++i) {
monitor.checkCanceled();
monitor.setProgress(i);
monitor.setProgress(
matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size())));
Match match = mymatches.get(i);
Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
if (!match.checkPostRules(streamoffset + blockOffset)) {
continue;
}
MatchAction[] matchactions = match.getMatchActions();
MatchAction[] matchactions = match.getMatchActions();
preMatchApply(matchactions, addr);
for (MatchAction matchaction : matchactions) {
matchaction.apply(program, addr, match);
}
postMatchApply(matchactions, addr);
}
numDone += numAddressesInRange;
}
}
/**
* Called before any match rules are applied
* @param matchactions actions that matched
* @param addr address of match
*/
public void preMatchApply(MatchAction[] matchactions, Address addr) {
// override if any initialization needs to be done before rule application
}
/**
* Called after any match rules are applied
* Can use for cross post rule matching state application and cleanup.
* @param matchactions actions that matched
* @param addr adress of match
*/
public void postMatchApply(MatchAction[] matchactions, Address addr) {
// override if any cleanup from rule match application is needed
}
}

View file

@ -37,6 +37,9 @@ public class Pattern extends DittedBitSequence {
private PostRule[] postrule;
private MatchAction[] actions;
/**
* Construct an empty pattern. Use XML to initialize
*/
public Pattern() {
markOffset = 0;
postrule = null;
@ -44,6 +47,15 @@ public class Pattern extends DittedBitSequence {
}
/**
* Construct the pattern based on a DittedByteSequence a match offset, post matching rules,
* and a set of actions to take when the match occurs.
*
* @param seq DittedByteSequence
* @param offset offset from the actual match location to report a match
* @param postArray post set of rules to check for the match
* @param matchArray MatchActions to apply when a match occurs
*/
public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray,
MatchAction[] matchArray) {
super(seq);

View file

@ -19,7 +19,18 @@ package ghidra.util.bytesearch;
* Interface for factories that create Match Pattern classes
*/
public interface PatternFactory {
/**
* Get a named match action
*
* @param nm name of action to find
* @return match action with the given name, null otherwise
*/
public MatchAction getMatchActionByName(String nm);
/**
* Get a named post match rule by name
* @param nm name of the post rule
* @return the post rule with the name, null otherwise
*/
public PostRule getPostRuleByName(String nm);
}

View file

@ -59,6 +59,9 @@ public class PatternPairSet {
private ArrayList<DittedBitSequence> preSequences;
private ArrayList<Pattern> postPatterns;
/**
* Construct an empty PatternPairSet. Use XML to initialize the pattern sets.
*/
public PatternPairSet() {
preSequences = new ArrayList<DittedBitSequence>();
postPatterns = new ArrayList<Pattern>();
@ -84,12 +87,22 @@ public class PatternPairSet {
}
}
/**
* Add this PatternPairSets post patterns to an existing arraylist of patterns.
* @param postpats array to add this PatternPairSets post patterns into
*/
public void extractPostPatterns(ArrayList<Pattern> postpats) {
for (int i = 0; i < postPatterns.size(); ++i) {
postpats.add(postPatterns.get(i));
}
}
/**
* Restore PatternPairSet from XML pull parser
* @param parser XML pull parser
* @param pfactory pattern factory user to construct patterns
* @throws IOException if pull parsing fails
*/
public void restoreXml(XmlPullParser parser, PatternFactory pfactory) throws IOException {
XmlElement el = parser.start("patternpairs");
totalBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("totalbits"));

View file

@ -21,7 +21,18 @@ import ghidra.xml.XmlPullParser;
* Inteface for post match rules that are checked after a match is idenfied
*/
public interface PostRule {
/**
* Apply a post rule given the matching pattern and offset into the byte stream.
* @param pat pattern that matched
* @param matchoffset offset of the match
* @return true if the PostRule is satisfied
*/
public boolean apply(Pattern pat, long matchoffset);
/**
* Can restore state of instance PostRule from XML
*
* @param parser XML pull parser
*/
public void restoreXml(XmlPullParser parser);
}

View file

@ -27,19 +27,27 @@ import ghidra.util.task.TaskMonitor;
*/
public class SequenceSearchState implements Comparable<SequenceSearchState> {
private static final int PATERN_ENDED = 10000000;
private static final int PATTERN_ENDED = Integer.MAX_VALUE;
private SequenceSearchState parent;
private ArrayList<DittedBitSequence> possible; // Patterns that could still match in this state
private ArrayList<DittedBitSequence> success; // Patterns that have matched successfully if we reached this state
private SequenceSearchState[] trans; // State transitions based on next byte
public SequenceSearchState(SequenceSearchState par) {
parent = par;
/**
* Construct a sub sequence state with a parent sequence
*
* @param parent parent SequenceSearchState
*/
public SequenceSearchState(SequenceSearchState parent) {
this.parent = parent;
possible = new ArrayList<DittedBitSequence>();
success = null;
trans = null;
}
/**
* @return maximum number of bytes that could be matched by this sequence
*/
public int getMaxSequenceSize() {
int max = 0;
for (DittedBitSequence element : possible) {
@ -51,6 +59,12 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
return max;
}
/**
* Add a pattern to this search sequence. The last pattern added is the successful
* match pattern.
* @param pat pattern to add
* @param pos position within the current set of patterns to add this pattern
*/
public void addSequence(DittedBitSequence pat, int pos) {
possible.add(pat);
if (pos == pat.getSize()) {
@ -61,6 +75,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
}
/**
* Sort the sequences that have been added
*/
public void sortSequences() {
Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
@Override
@ -150,9 +167,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
i += 1;
j += 1;
thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
oppat =
(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
}
else if (thispat < oppat) {
if (curpat != thispat) {
@ -160,7 +177,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
curpat = thispat;
}
i += 1;
thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
}
else {
if (curpat != oppat) {
@ -169,7 +186,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
j += 1;
oppat =
(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
}
}
success = tmp;
@ -177,6 +194,12 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
}
/**
* Try to match this Sequence to the byteArray, and add any matches to the match list
* @param bytearray array of bytes to match
* @param numbytes retrict number of bytes to allow to match
* @param match list of matches, the result
*/
public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList<Match> match) {
int subindex = 0;
SequenceSearchState curstate = this;
@ -242,6 +265,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
*/
public void apply(InputStream in, long maxBytes, ArrayList<Match> match, TaskMonitor monitor)
throws IOException {
long progress = monitor.getProgress();
int maxSize = getMaxSequenceSize() + 1;
if (maxSize < 4096) {
maxSize = 4096;
@ -317,7 +342,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
if (monitor.isCancelled()) {
return;
}
monitor.setProgress(offset);
monitor.setProgress(progress + offset);
}
if (ra != secondBuf.length) {
fullBuffers = 1;
@ -373,8 +398,15 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
}
static public ArrayList<SequenceSearchState> buildTransitionLevel(
ArrayList<SequenceSearchState> prev, int pos) {
/**
* Build a new transition level for the state machine
*
* @param prev previous search sequences
* @param pos position within the search sequence state for this level
* @return list of possible new search states to be added to the state machine
*/
static ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,
int pos) {
ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iterator = prev.iterator();
while (iterator.hasNext()) { // For each current state
@ -407,6 +439,11 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
return finalres;
}
/**
* Build a search state machine from a list of DittedBitSequences
* @param patterns bit sequence patterns
* @return search state the will match the given sequences
*/
static public SequenceSearchState buildStateMachine(
ArrayList<? extends DittedBitSequence> patterns) {
SequenceSearchState root = new SequenceSearchState(null);