diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java index 3bfeeb886a..a49b3438a3 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java @@ -15,7 +15,8 @@ */ package ghidra.app.plugin.core.analysis; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import ghidra.app.cmd.data.CreateDataCmd; import ghidra.app.services.*; @@ -23,15 +24,17 @@ import ghidra.app.util.importer.MessageLog; import ghidra.framework.options.Options; import ghidra.program.model.address.*; import ghidra.program.model.data.*; -import ghidra.program.model.listing.*; +import ghidra.program.model.listing.BookmarkType; +import ghidra.program.model.listing.Program; import ghidra.program.model.mem.Memory; +import ghidra.util.bytesearch.*; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; public class EmbeddedMediaAnalyzer extends AbstractAnalyzer { private static final String NAME = "Embedded Media"; private static final String DESCRIPTION = - "Finds and tries to apply embedded media data types (ie png, gif, jpeg, wav) in current program."; + "Finds embedded media data types (ie png, gif, jpeg, wav)"; private static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks"; private static final String OPTION_DESCRIPTION_CREATE_BOOKMARKS = @@ -52,92 +55,75 @@ public class EmbeddedMediaAnalyzer extends AbstractAnalyzer { throws CancelledException { Memory memory = program.getMemory(); - AddressSetView initializedAddressSet = memory.getLoadedAndInitializedAddressSet(); - AddressSet initialedSearchSet = set.intersect(initializedAddressSet); + AddressSetView validMemorySet = memory.getLoadedAndInitializedAddressSet(); + AddressSet searchSet = set.intersect(validMemorySet); + + MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("Embedded Media"); List
foundMedia = new ArrayList<>(); - foundMedia = scanForMedia(program, new GifDataType(), "GIF 87", GifDataType.MAGIC_87, - GifDataType.GIFMASK, initialedSearchSet, memory, monitor); + addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 87", + GifDataType.MAGIC_87, GifDataType.GIFMASK); - foundMedia.addAll(scanForMedia(program, new GifDataType(), "GIF 89", GifDataType.MAGIC_89, - GifDataType.GIFMASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 89", + GifDataType.MAGIC_89, GifDataType.GIFMASK); - foundMedia.addAll(scanForMedia(program, new PngDataType(), "PNG", PngDataType.MAGIC, - PngDataType.MASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new PngDataType(), "PNG", + PngDataType.MAGIC, PngDataType.MASK); - foundMedia.addAll(scanForMedia(program, new JPEGDataType(), "JPEG", JPEGDataType.MAGIC, - JPEGDataType.MAGIC_MASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new JPEGDataType(), "JPEG", + JPEGDataType.MAGIC, JPEGDataType.MAGIC_MASK); - foundMedia.addAll(scanForMedia(program, new WAVEDataType(), "WAVE", WAVEDataType.MAGIC, - WAVEDataType.MAGIC_MASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new WAVEDataType(), "WAVE", + WAVEDataType.MAGIC, WAVEDataType.MAGIC_MASK); - foundMedia.addAll(scanForMedia(program, new AUDataType(), "AU", AUDataType.MAGIC, - AUDataType.MAGIC_MASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new AUDataType(), "AU", + AUDataType.MAGIC, AUDataType.MAGIC_MASK); - foundMedia.addAll(scanForMedia(program, new AIFFDataType(), "AIFF", AIFFDataType.MAGIC, - AIFFDataType.MAGIC_MASK, initialedSearchSet, memory, monitor)); + addByteSearchPattern(searcher, program, foundMedia, new AIFFDataType(), "AIFF", + AIFFDataType.MAGIC, AIFFDataType.MAGIC_MASK); - return true; + searcher.search(program, searchSet, monitor); + + return foundMedia.size() > 0; } - private List
scanForMedia(Program program, DataType dt, String mediaName, - byte[] mediaBytes, byte[] mask, AddressSetView addresses, Memory memory, - TaskMonitor monitor) { - - monitor.setMessage("Scanning for " + mediaName + " Embedded Media"); - monitor.initialize(addresses.getNumAddresses()); - - List
foundMediaAddresses = new ArrayList<>(); - - Iterator iterator = addresses.iterator(); - while (iterator.hasNext()) { - if (monitor.isCancelled()) { - return foundMediaAddresses; - } - - AddressRange range = iterator.next(); - Address start = range.getMinAddress(); - Address end = range.getMaxAddress(); - - Address found = memory.findBytes(start, end, mediaBytes, mask, true, monitor); - while (found != null && !monitor.isCancelled()) { - //See if it is already an applied media data type - Data data = program.getListing().getDefinedDataAt(found); - int skipLen = 1; - if (data == null) { - try { - CreateDataCmd cmd = new CreateDataCmd(found, dt); - if (cmd.applyTo(program)) { - if (createBookmarksEnabled) { - program.getBookmarkManager().setBookmark(found, - BookmarkType.ANALYSIS, "Embedded Media", - "Found " + mediaName + " Embedded Media"); - } - foundMediaAddresses.add(found); - //have to get the actual applied data to find the actual length to skip because until then it can't compute the length due to the data type being dynamic - skipLen = program.getListing().getDataAt(found).getLength(); - } - } - //If media does not apply correctly then it is not really a that media data type or there is other data in the way - catch (Exception e) { - // Not a valid embedded media or no room to apply it so just ignore it and skip it - } - } - // skip either the valid data that was found or skip one byte - // then do the next search - try { - start = found.add(skipLen); - found = memory.findBytes(start, end, mediaBytes, mask, true, monitor); - } - catch (AddressOutOfBoundsException e) { - // If media was at the very end of the address space, we will end up here - break; - } - } + private void addByteSearchPattern(MemoryBytePatternSearcher searcher, Program program, + List
foundMedia, DataType mediaDT, String mediaName, byte[] bytes, + byte[] mask) { + if (bytes == null) { + return; } - return foundMediaAddresses; + GenericMatchAction action = new GenericMatchAction(mediaDT) { + @Override + public void apply(Program prog, Address addr, Match match) { + //See if it is already an applied media data type + if (!program.getListing().isUndefined(addr, addr)) { + return; + } + + try { + CreateDataCmd cmd = new CreateDataCmd(addr, mediaDT); + if (cmd.applyTo(program)) { + if (createBookmarksEnabled) { + program.getBookmarkManager().setBookmark(addr, BookmarkType.ANALYSIS, + "Embedded Media", "Found " + mediaName + " Embedded Media"); + } + foundMedia.add(addr); + } + } + //If media does not apply correctly then it is not really a that media data type or there is other data in the way + catch (Exception e) { + // Not a valid embedded media or no room to apply it so just ignore it and skip it + } + } + }; + + GenericByteSequencePattern genericByteMatchPattern = + new GenericByteSequencePattern(bytes, mask, action); + + searcher.addPattern(genericByteMatchPattern); } @Override diff --git a/Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java b/Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java index f87cc044b4..fb4e1e4276 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java @@ -177,8 +177,7 @@ public class ProgramMemoryUtil { MemoryBlock[] tmpBlocks = new MemoryBlock[blocks.length]; int j = 0; for (MemoryBlock block : blocks) { - if ((block.isInitialized() && withBytes) || - (!block.isInitialized() && !withBytes)) { + if ((block.isInitialized() && withBytes) || (!block.isInitialized() && !withBytes)) { tmpBlocks[j++] = block; } } @@ -493,6 +492,30 @@ public class ProgramMemoryUtil { monitor = TaskMonitorAdapter.DUMMY_MONITOR; } + byte[] addressBytes = getDirectAddressBytes(program, toAddress); + + byte[] shiftedAddressBytes = getShiftedDirectAddressBytes(program, toAddress); + + Memory memory = program.getMemory(); + Set
dirRefsAddrs = new TreeSet<>(); + findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor); + + if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory + findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor); + } + + return dirRefsAddrs; + } + + /** + * Get a representation of an address as it would appear in bytes in memory. + * + * @param program program + * @param toAddress target address + * @return byte representation of toAddress + */ + public static byte[] getDirectAddressBytes(Program program, Address toAddress) { + Memory memory = program.getMemory(); boolean isBigEndian = memory.isBigEndian(); @@ -536,6 +559,31 @@ public class ProgramMemoryUtil { addressBytes, 0, addressBytes.length); } + return addressBytes; + } + + /** + * returns shifted address bytes if they are different than un-shifted + * + * @param program program + * @param toAddress target address + * @return shifted bytes, null if same as un-shifted + */ + public static byte[] getShiftedDirectAddressBytes(Program program, Address toAddress) { + + byte[] addressBytes = getDirectAddressBytes(program, toAddress); + + Memory memory = program.getMemory(); + boolean isBigEndian = memory.isBigEndian(); + + DataConverter dataConverter; + if (isBigEndian) { + dataConverter = new BigEndianDataConverter(); + } + else { + dataConverter = new LittleEndianDataConverter(); + } + byte[] shiftedAddressBytes = null; DataTypeManager dataTypeManager = program.getDataTypeManager(); DataOrganization dataOrganization = dataTypeManager.getDataOrganization(); @@ -554,24 +602,23 @@ public class ProgramMemoryUtil { } } - // don't need this anymore - finding all 16 bit addrs in whole prog -// AddressRange segmentRange = null; -// if (toAddress instanceof SegmentedAddress) { -// // Restrict search to currentSegment range -// SegmentedAddressSpace segSpace = (SegmentedAddressSpace) toAddress.getAddressSpace(); -// segmentRange = -// new AddressRangeImpl(segSpace.getAddress(currentSegment, 0), segSpace.getAddress( -// currentSegment, 0xffff)); -// } + return shiftedAddressBytes; + } - Set
dirRefsAddrs = new TreeSet<>(); - findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor); + public static byte[] getImageBaseOffsets32Bytes(Program program, int alignment, + Address toAddress) { - if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory - findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor); + Address imageBase = program.getImageBase(); + + long offsetValue = toAddress.subtract(imageBase); + int offsetSize = 4; // 32 bit offset + byte[] bytes = new byte[offsetSize]; + for (int i = 0; i < offsetSize; i++) { + bytes[i] = (byte) offsetValue; + offsetValue >>= 8; // Shift by a single byte. } - return dirRefsAddrs; + return bytes; } /** @@ -627,8 +674,7 @@ public class ProgramMemoryUtil { if (!block.isInitialized()) { continue; } - if (memoryRange != null && - !memoryRange.intersects(block.getStart(), block.getEnd())) { + if (memoryRange != null && !memoryRange.intersects(block.getStart(), block.getEnd())) { // skip blocks which do not correspond to currentSeg continue; } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java similarity index 59% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java index a1e9f21394..314c57b43e 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java @@ -19,24 +19,43 @@ import ghidra.util.xml.SpecXmlUtils; import ghidra.xml.XmlElement; import ghidra.xml.XmlPullParser; +/** + * ByteSearch post search rule when a pattern is found, The pattern must have a certain + * alignment at an offset from the location the pattern matches. The alignment is + * specified by the mask bits that must be zero. + * + * mark is the offset in bytes from the start of the matching pattern. + * + * align 2 = 0x1 - lower bit must be zero + * align 4 = 0x3 - lower two bits must be zero + * align 8 = 0x7 - lower three bits must be zero + * align 16 = 0xF - lower four bits must be zero + * .... + * Other strange alignments could be specified, but most likely the above suffice. + * + * The pattern can be constructed or restored from XML of the form: + * + * + * + */ + public class AlignRule implements PostRule { private int mark; // Position, relative to start of pattern, to check alignment at private int alignmask; // Mask of bits that must be zero - - - public AlignRule(){ + + public AlignRule() { } - - public AlignRule(int mark, int alignmask){ + + public AlignRule(int mark, int alignmask) { this.mark = mark; this.alignmask = alignmask; } - + @Override public boolean apply(Pattern pat, long matchoffset) { - int off = (int)matchoffset; - return (((off + mark) & alignmask)==0); + int off = (int) matchoffset; + return (((off + mark) & alignmask) == 0); } @Override @@ -44,7 +63,7 @@ public class AlignRule implements PostRule { XmlElement el = parser.start("align"); mark = SpecXmlUtils.decodeInt(el.getAttribute("mark")); int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits")); - alignmask = (1<= bits.length) { return false; @@ -187,14 +221,38 @@ public class DittedBitSequence { return ((byte) (val & dits[pos])) == bits[pos]; } + /** + * Set a an index in a larger sequence, or identifing id on this pattern + * + * @param index - index in match sequence, or unique id + */ + public void setIndex(int index) { + this.index = index; + } + + /** + * Get the index or identifying id attached to this pattern + * + * @return index or unique id attached to this sequence + */ public int getIndex() { return index; } + /** + * get the size of this sequence in bytes + * + * @return size in bytes + */ public int getSize() { return bits.length; } + /** + * Get number of bits that must be 0/1 + * + * @return number of bits that are not don't care (ditted) + */ public int getNumFixedBits() { int popcnt = 0; for (byte dit : dits) { @@ -203,7 +261,11 @@ public class DittedBitSequence { return popcnt; } - //Return the number of dits. + /** + * Get number of bits that are ditted (don't care) + * + * @return number of ditted bits (don't care) + */ public int getNumUncertainBits() { int popcnt = 0; for (byte dit : dits) { @@ -235,6 +297,11 @@ public class DittedBitSequence { return buf.toString(); } + /** + * get a ditted hex string representing this sequence + * + * @return ditted hex string + */ public String getHexString() { String uncompressed = this.toString(); String[] parts = uncompressed.trim().split(" "); @@ -260,6 +327,17 @@ public class DittedBitSequence { return sb.toString(); } + /** + * restore ditted string from XML stream with hex/binary ditted sequences in the form: + * 0x..d.4de2 ....0000 .1...... 00101101 11101001 + * where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary) + * + * @param parser XML pull parser stream + * + * @return number of bytes read from XML tag + * + * @throws IOException if XML read has an error + */ protected int restoreXmlData(XmlPullParser parser) throws IOException { parser.start("data"); String text = parser.end().getText(); @@ -272,6 +350,16 @@ public class DittedBitSequence { } } + /** + * Initialize this sequence with a ditted sequence from a string in the form + * (e.g. - 011...1., 0x.F, 01110011 0xAB) + * + * @param text ditted sequence + * + * @return number of bytes in the ditted sequence + * + * @throws IllegalArgumentException if string is malformed + */ private int initFromDittedStringData(String text) throws IllegalArgumentException { int markOffset = -1; int mode = -1; // -1: looking for start, -2: skip to EOL, 0: hex mode, 1: binary mode @@ -372,6 +460,13 @@ public class DittedBitSequence { return markOffset; } + /** + * Get the number of bits that are fixed, not ditted (don't care) + * + * @param marked number of bytes in the pattern to check + * + * @return number of initial fixed bits + */ public int getNumInitialFixedBits(int marked) { if (dits == null) { return 0; @@ -385,5 +480,4 @@ public class DittedBitSequence { } return popcnt; } - } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java similarity index 91% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java index 29aa9689a0..e8e58e4c37 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java @@ -1,6 +1,5 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +19,9 @@ import ghidra.program.model.address.Address; import ghidra.program.model.listing.Program; import ghidra.xml.XmlPullParser; +/** + * Dummy action attached to a match sequence. Action is not restored from XML + */ public class DummyMatchAction implements MatchAction { @Override @@ -30,6 +32,5 @@ public class DummyMatchAction implements MatchAction { public void restoreXml(XmlPullParser parser) { parser.discardSubTree(); } - -} +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java new file mode 100644 index 0000000000..773dd04208 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java @@ -0,0 +1,46 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +import ghidra.program.model.data.DataType; + +/** + * Templated simple DittedBitSequence Pattern for a byte/mask pattern and associated action. + * The DittedBitSequence is provided by value and mask in byte arrays. + * + * This class is normally used to find some number of SequencePatterns within a seqence of bytes. + * When the byte/mask pattern is matched, the GenericMatchAction will be "applied". + * + * @param the class of match action, used to specify a specialized momento to be used by the action when it is "applied". + */ + +public class GenericByteSequencePattern extends Pattern { + + public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction action) { + super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]); + + MatchAction[] matchActions = getMatchActions(); + matchActions[0] = action; + } + + public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask, + GenericMatchAction action) { + super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]); + + MatchAction[] matchActions = getMatchActions(); + matchActions[0] = action; + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java new file mode 100644 index 0000000000..7155e981fb --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java @@ -0,0 +1,42 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +/** + * Template for generic match action attached to a match sequence. + * Used to store an associated value to the matching sequence. + * The associated value can be retrieved when the sequence is matched. + * + * @param - object to attach to match sequence, generally used to specify + * a specialized momento to be used by the action when it is "applied". + */ +public class GenericMatchAction extends DummyMatchAction { + T matchValue; + + /* + * construct with an appropriate object that can be used when the action is applied + */ + public GenericMatchAction(T matchValue) { + this.matchValue = matchValue; + } + + /** + * @return the specialized object associated with this match action + */ + public T getMatchValue() { + return this.matchValue; + } +} diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Match.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java similarity index 63% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Match.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java index fb48f8bb07..7da007cbe4 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Match.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java @@ -15,13 +15,21 @@ */ package ghidra.util.bytesearch; +/** + * Represents a match of a DittedBitSequence at a given offset in a byte sequence. + * + * There is a hidden assumption that the sequence is actually a Pattern + * that might have a ditted-bit-sequence, a set of match actions, + * and post match rules/checks + * + */ public class Match { private DittedBitSequence sequence; // Pattern that matches private long offset; // starting offset within bytestream of match - public Match(DittedBitSequence seq, long off) { - sequence = seq; - offset = off; + public Match(DittedBitSequence sequence, long offset) { + this.sequence = sequence; + this.offset = offset; } /** @@ -39,42 +47,70 @@ public class Match { return sequence.getNumFixedBits() - sequence.getNumInitialFixedBits(marked); } + /** + * @return actions associated with this match + */ public MatchAction[] getMatchActions() { return ((Pattern) sequence).getMatchActions(); } + /** + * @return size in bytes of sequence + */ public int getSequenceSize() { return sequence.getSize(); } + /** + * @return index of sequence in a possibly longer set of sequences + */ public int getSequenceIndex() { return sequence.getIndex(); } + /** + * @return the offset of the match within a longer byte sequence + */ public long getMarkOffset() { return offset + ((Pattern) sequence).getMarkOffset(); } + /** + * @return offset of match in sequence of bytes + */ public long getMatchStart() { return offset; } + /** + * Check that the possible post rules are satisfied + * + * @param streamoffset offset within from match location to check postrules. + * + * @return true if post rules are satisfied + */ public boolean checkPostRules(long streamoffset) { long curoffset = streamoffset + offset; Pattern pattern = (Pattern) sequence; PostRule[] postRules = pattern.getPostRules(); - for (int i = 0; i < postRules.length; ++i) { - if (!postRules[i].apply(pattern, curoffset)) { + for (PostRule postRule : postRules) { + if (!postRule.apply(pattern, curoffset)) { return false; } } return true; } + /** + * @return ditted bit sequence as a string + */ public String getHexString() { return sequence.getHexString(); } + /** + * @return the sequence that was matched + */ public DittedBitSequence getSequence() { return sequence; } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/MatchAction.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java similarity index 83% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/MatchAction.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java index 1ef3b59008..750aedeb0f 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/MatchAction.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java @@ -1,6 +1,5 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,11 +20,10 @@ import ghidra.program.model.listing.Program; import ghidra.xml.XmlPullParser; /** - * Action that should be applied to a Program at the Address a pattern matches - * + * Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern */ public interface MatchAction { - public void apply(Program program,Address addr,Match match); - + public void apply(Program program, Address addr, Match match); + public void restoreXml(XmlPullParser parser); } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java new file mode 100644 index 0000000000..6683156057 --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java @@ -0,0 +1,179 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.util.bytesearch; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; + +import ghidra.program.model.address.*; +import ghidra.program.model.listing.Program; +import ghidra.program.model.mem.MemoryBlock; +import ghidra.util.Msg; +import ghidra.util.exception.CancelledException; +import ghidra.util.task.TaskMonitor; + +/** + * Multi pattern/mask/action memory searcher + * + * Preload search patterns and actions, then call search method. + */ + +public class MemoryBytePatternSearcher { + private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32; + + ArrayList patternList; + + private String searchName = ""; + + /** + * Create with pre-created patternList + * + * @param patternList - list of patterns(bytes/mask/action) + */ + public MemoryBytePatternSearcher(String searchName, ArrayList patternList) { + this.searchName = searchName; + this.patternList = new ArrayList(patternList); + } + + /** + * Create with no patternList, must add patterns before searching + * + */ + public MemoryBytePatternSearcher(String searchName) { + this.searchName = searchName; + patternList = new ArrayList<>(); + } + + /** + * Add a search pattern + * @param pattern - pattern(bytes/mask/action) + */ + public void addPattern(Pattern pattern) { + patternList.add(pattern); + } + + /** + * Search initialized memory blocks for all patterns(bytes/mask/action). + * Call associated action for each pattern matched. + * + * @param program to be searched + * @param searchSet set of bytes to restrict search to + * @param monitor allow canceling and reporting of progress + * + * @throws CancelledException if canceled + */ + public void search(Program program, AddressSetView searchSet, TaskMonitor monitor) + throws CancelledException { + SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList); + + MemoryBlock[] blocks = program.getMemory().getBlocks(); + for (MemoryBlock block2 : blocks) { + MemoryBlock block = block2; + if (!searchSet.intersects(block.getStart(), block.getEnd())) { + continue; + } + try { + searchBlock(root, program, block, searchSet, monitor); + } + catch (IOException e) { + Msg.error(this, "Unable to scan block " + block.getName() + " for patterns"); + } + } + } + + /** + * Search through bytes of a memory block using the finite state machine -root- + * Apply any additional rules for matching patterns. + * + * @param program is the Program being searched + * @param block is the specific block of bytes being searched + * + * @throws IOException exception during read of memory + * @throws CancelledException canceled search + */ + private void searchBlock(SequenceSearchState rootState, Program program, MemoryBlock block, + AddressSetView restrictSet, TaskMonitor monitor) + throws IOException, CancelledException { + + // if no restricted set, make restrict set the full block + AddressSet doneSet = new AddressSet(restrictSet); + if (doneSet.isEmpty()) { + doneSet.addRange(block.getStart(), block.getEnd()); + } + doneSet = doneSet.intersectRange(block.getStart(), block.getEnd()); + + Address blockStartAddr = block.getStart(); + + // pull each range off the restricted set + AddressRangeIterator addressRanges = doneSet.getAddressRanges(); + while (addressRanges.hasNext()) { + monitor.checkCanceled(); + AddressRange addressRange = addressRanges.next(); + + monitor.setMessage(searchName + " Search"); + monitor.initialize(doneSet.getNumAddresses()); + monitor.setProgress(0); + + ArrayList mymatches = new ArrayList<>(); + + long streamoffset = blockStartAddr.getOffset(); + + // Give block a starting/ending point before this address to search + // patterns might start before, since they have a pre-pattern + // TODO: this is dangerous, since pattern might be very big, but the set should be restricted + // normally only when we are searching for more matching patterns that had a postrule that didn't satisfy + // normally the whole memory blocks will get searched. + long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr); + blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE; + if (blockOffset <= 0) { + // don't go before the block start + blockOffset = 0; + } + + // compute number of bytes in the range + 1, and don't search more than that. + long maxBlockSearchLength = + addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1; + + InputStream data = block.getData(); + data.skip(blockOffset); + + rootState.apply(data, maxBlockSearchLength, mymatches, monitor); + monitor.checkCanceled(); + + monitor.setMessage(searchName + " (Examine Matches)"); + monitor.initialize(mymatches.size()); + monitor.setProgress(0); + + // TODO: DANGER there is much offset<-->address calculation here + // should be OK, since they are all relative to the block. + for (int i = 0; i < mymatches.size(); ++i) { + monitor.checkCanceled(); + monitor.setProgress(i); + Match match = mymatches.get(i); + Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset); + if (!match.checkPostRules(streamoffset + blockOffset)) { + continue; + } + MatchAction[] matchactions = match.getMatchActions(); + + for (MatchAction matchaction : matchactions) { + matchaction.apply(program, addr, match); + } + } + } + } +} diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Pattern.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java similarity index 94% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Pattern.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java index c158590b58..1b7dc035a1 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Pattern.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java @@ -24,6 +24,13 @@ import generic.jar.ResourceFile; import ghidra.util.xml.SpecXmlUtils; import ghidra.xml.*; +/** + * Pattern is an association of a DittedBitSequence to match, + * a set of post rules after a match is found that must be satisfied, + * and a set of actions to be taken if the pattern matches. + * + * These patterns can be restored from an XML file. + */ public class Pattern extends DittedBitSequence { private int markOffset; // Within pattern what is the 'marked' byte @@ -37,7 +44,8 @@ public class Pattern extends DittedBitSequence { } - public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, MatchAction[] matchArray) { + public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, + MatchAction[] matchArray) { super(seq); markOffset = offset; postrule = postArray; @@ -51,8 +59,8 @@ public class Pattern extends DittedBitSequence { public MatchAction[] getMatchActions() { return actions; } - - public void setMatchActions(MatchAction[] actions){ + + public void setMatchActions(MatchAction[] actions) { this.actions = actions; } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternFactory.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java similarity index 91% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternFactory.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java index f37b22beb6..47b29373da 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternFactory.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java @@ -1,6 +1,5 @@ /* ### * IP: GHIDRA - * REVIEWED: YES * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,8 +15,11 @@ */ package ghidra.util.bytesearch; +/** + * Interface for factories that create Match Pattern classes + */ public interface PatternFactory { public MatchAction getMatchActionByName(String nm); - + public PostRule getPostRuleByName(String nm); } diff --git a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternPairSet.java b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java similarity index 62% rename from Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternPairSet.java rename to Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java index 9741d87d7c..8d621220ef 100644 --- a/Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternPairSet.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java @@ -23,66 +23,91 @@ import ghidra.xml.XmlElement; import ghidra.xml.XmlPullParser; /** - * Two collections of patterns that are paired together to create larger patterns - * The final large patterns all must first match a pattern from the "pre" pattern collection - * followed immediately by a pattern from the "post" pattern collection + * A set of "pre" DittedBitSequences and a set of "post" Patterns are paired to form a larger pattern. + * To match, a sequence from the "pre" sequence set must first match, then one of the "post" patterns + * is matched relative to the matching "pre" pattern. This class is really a storage object for the + * patterns and provides a mechanism to read the pre/post patterns from an XML file. * + * The larger pattern has the idea of bits of check, which means the number of bits that are fixed to + * a value when matching (not don't care). There is a pre pattern bits of check and post pattern bits + * of check. The bits of check are used to statistically gauge the accuracy of the pattern. + * + * An example of the XML format follows: + * + * + * 0xe12fff1. + * 0xe12fff1e 0x46c0 + * 0xe12fff1e 0xe1a00000 + * + * + * + * 0xe24dd... 11101001 00101101 .1...... ....0000 + * 11101001 00101101 .1...... ....0000 0xe24dd... + * 11101001 00101101 .1...... ....0000 0x........ 0xe24dd... + * + * + * + * + * + * + * Note: The post Patterns can also have a set of rules that must be satisfied along with one of the + * Pattern DittedBitSequence matches. */ public class PatternPairSet { private int totalBitsOfCheck; // Minimum number of bits of check in final patterns private int postBitsOfCheck; // Minimum bits of check in "post" part of pattern private ArrayList preSequences; private ArrayList postPatterns; - + public PatternPairSet() { preSequences = new ArrayList(); postPatterns = new ArrayList(); } - + public void createFinalPatterns(ArrayList finalpats) { - for(int i=0;i postpats) { - for(int i=0;i postdit = new ArrayList(); - while(el.isStart() && el.getName().equals("data")) { + while (el.isStart() && el.getName().equals("data")) { DittedBitSequence postseq = new DittedBitSequence(); postseq.restoreXmlData(parser); if (postseq.getNumFixedBits() >= postBitsOfCheck) { @@ -99,8 +124,8 @@ public class PatternPairSet { postRuleArray.toArray(postRules); MatchAction[] matchActions = new MatchAction[matchActionArray.size()]; matchActionArray.toArray(matchActions); - for(int i=0;i { - + + private static final int PATERN_ENDED = 10000000; private SequenceSearchState parent; private ArrayList possible; // Patterns that could still match in this state private ArrayList success; // Patterns that have matched successfully if we reached this state private SequenceSearchState[] trans; // State transitions based on next byte - + public SequenceSearchState(SequenceSearchState par) { parent = par; possible = new ArrayList(); @@ -40,23 +42,25 @@ public class SequenceSearchState implements Comparable { public int getMaxSequenceSize() { int max = 0; - for(int i=0;i max) + for (DittedBitSequence element : possible) { + int val = element.getSize(); + if (val > max) { max = val; + } } return max; } - - public void addSequence(DittedBitSequence pat,int pos) { + + public void addSequence(DittedBitSequence pat, int pos) { possible.add(pat); if (pos == pat.getSize()) { - if (success == null) + if (success == null) { success = new ArrayList(); + } success.add(pat); } } - + public void sortSequences() { Comparator comp = new Comparator() { @Override @@ -64,38 +68,42 @@ public class SequenceSearchState implements Comparable { return o1.getIndex() - o2.getIndex(); } }; - Collections.sort(possible,comp); - if (success != null) - Collections.sort(success,comp); + Collections.sort(possible, comp); + if (success != null) { + Collections.sort(success, comp); + } } - + @Override public int compareTo(SequenceSearchState o) { - int i=0; - for(;;) { + int i = 0; + for (;;) { if (possible.size() <= i) { - if (o.possible.size() <=i) + if (o.possible.size() <= i) { return 0; + } return -1; } - if (o.possible.size() <= i) + if (o.possible.size() <= i) { return 1; + } int indus = possible.get(i).getIndex(); // Lexicographic compare an sequence of sequences int indthem = o.possible.get(i).getIndex(); - if (indus != indthem) + if (indus != indthem) { return (indus < indthem) ? -1 : 1; + } i += 1; } } - - private void buildSingleTransition(ArrayList all,int pos,int val) { + + private void buildSingleTransition(ArrayList all, int pos, int val) { SequenceSearchState newstate = null; - for(int i=0;i { all.add(newstate); } } - - private void exportSuccess(ArrayList match,int offset) { - for(int i=0;i match, int offset) { + for (DittedBitSequence succes : success) { // If we found matches + Match newmatch = new Match(succes, offset); match.add(newmatch); - } + } } - + /** * Merge in -op- and this as a single state * @param op */ private void merge(SequenceSearchState op) { SequenceSearchState parent = op.parent; - for(int i=0;i<256;++i) { - if (parent.trans[i] == op) // Any references to -op- in parent + for (int i = 0; i < 256; ++i) { + if (parent.trans[i] == op) { parent.trans[i] = this; // Should be replaced with this + } } if (op.success != null) { // Merge if (success == null) { @@ -128,12 +137,12 @@ public class SequenceSearchState implements Comparable { } else { ArrayList tmp = new ArrayList(); - int i=0; - int j=0; + int i = 0; + int j = 0; int curpat = -1; - int thispat = success.get(i).index; - int oppat = op.success.get(j).index; - while((i { } i += 1; j += 1; - thispat = (i==success.size()) ? 10000000 : success.get(i).index; - oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index; + thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex(); + oppat = + (j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex(); } else if (thispat < oppat) { if (curpat != thispat) { @@ -150,7 +160,7 @@ public class SequenceSearchState implements Comparable { curpat = thispat; } i += 1; - thispat = (i==success.size()) ? 10000000 : success.get(i).index; + thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex(); } else { if (curpat != oppat) { @@ -158,26 +168,31 @@ public class SequenceSearchState implements Comparable { curpat = oppat; } j += 1; - oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index; + oppat = + (j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex(); } } success = tmp; } } } - - public void sequenceMatch(byte[] bytearray,int numbytes,ArrayList match) { + + public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList match) { int subindex = 0; SequenceSearchState curstate = this; - + do { - if (curstate.success != null) + if (curstate.success != null) { curstate.exportSuccess(match, 0); - if (subindex >= numbytes) return; - curstate = curstate.trans[ 0xff & bytearray[subindex] ]; // Perform state transition based on next byte in buffer + } + if (subindex >= numbytes) { + return; + } + curstate = curstate.trans[0xff & bytearray[subindex]]; // Perform state transition based on next byte in buffer subindex += 1; - } while(curstate != null); - + } + while (curstate != null); + } /** @@ -185,22 +200,24 @@ public class SequenceSearchState implements Comparable { * @param buffer is the array of bytes to search * @param match is populated with a Match object for each pattern and position that matches */ - public void apply(byte[] buffer,ArrayList match) { + public void apply(byte[] buffer, ArrayList match) { SequenceSearchState curstate; int subindex; - for(int offset=0;offset Root state subindex = offset; do { - if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point + if (curstate.success != null) { curstate.exportSuccess(match, offset); + } if (subindex >= buffer.length) { // if we've run out of bytes, must restart at next offset break; } - curstate = curstate.trans[ 0xff & buffer[subindex] ]; // Perform state transition based on next byte + curstate = curstate.trans[0xff & buffer[subindex]]; // Perform state transition based on next byte subindex += 1; - } while(curstate != null); - } + } + while (curstate != null); + } } /** @@ -210,10 +227,11 @@ public class SequenceSearchState implements Comparable { * @param monitor - if non-null, check for user cancel, and maintain progress info * @throws IOException */ - public void apply(InputStream in,ArrayList match,TaskMonitor monitor) throws IOException { - apply(in,-1L,match,monitor); + public void apply(InputStream in, ArrayList match, TaskMonitor monitor) + throws IOException { + apply(in, -1L, match, monitor); } - + /** * Search for pattern in the stream -in-. * @param in - The stream to scan for matches @@ -222,136 +240,160 @@ public class SequenceSearchState implements Comparable { * @param monitor - if non-null, check for user cancel, and maintain progress info * @throws IOException */ - public void apply(InputStream in, long maxBytes, ArrayList match,TaskMonitor monitor) throws IOException { - int maxsize = getMaxSequenceSize()+1; - if (maxsize <4096) - maxsize = 4096; - if (maxBytes > 0) { - maxBytes += getMaxSequenceSize()+1; + public void apply(InputStream in, long maxBytes, ArrayList match, TaskMonitor monitor) + throws IOException { + int maxSize = getMaxSequenceSize() + 1; + if (maxSize < 4096) { + maxSize = 4096; } - byte[] firstbuf=new byte[maxsize]; - byte[] secondbuf=new byte[maxsize]; - byte[] curbuf; - SequenceSearchState curstate; - int fullbuffers; // Number of buffers that are completely full - int ra = in.read(firstbuf); - if (ra == firstbuf.length) { - ra = in.read(secondbuf); - if (ra == secondbuf.length) { - fullbuffers = 2; + if (maxBytes > 0) { + maxBytes += getMaxSequenceSize() + 1; + } + byte[] firstBuf = new byte[maxSize]; + byte[] secondBuf = new byte[maxSize]; + byte[] curBuf; + SequenceSearchState curState; + int fullBuffers; // Number of buffers that are completely full + int ra = in.read(firstBuf); + if (ra == firstBuf.length) { + ra = in.read(secondBuf); + if (ra == secondBuf.length) { + fullBuffers = 2; } else { - if (ra < 0) + if (ra < 0) { ra = 0; - fullbuffers = 1; + } + fullBuffers = 1; byte[] tmp = new byte[ra]; - for(int i=0;i Root state - subindex = bufreloff; - curbuf = firstbuf; + int offset = 0; + int bufRelativeOffset = 0; + int subIndex; + while (fullBuffers == 2) { + curState = this; // New starting offset -> Root state + subIndex = bufRelativeOffset; + curBuf = firstBuf; do { - if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point - curstate.exportSuccess(match, offset); - if (subindex >= curbuf.length) { // check that we have enough bytes in current buffer - curbuf = secondbuf; // If not, switch to secondary buffer - subindex = 0; + if (curState.success != null) { + curState.exportSuccess(match, offset); } - curstate = curstate.trans[ 0xff & curbuf[subindex] ]; // Perform state transition based on next byte in buffer - subindex += 1; - } while(curstate != null); + if (subIndex >= curBuf.length) { // check that we have enough bytes in current buffer + curBuf = secondBuf; // If not, switch to secondary buffer + subIndex = 0; + } + curState = curState.trans[0xff & curBuf[subIndex]]; // Perform state transition based on next byte in buffer + subIndex += 1; + } + while (curState != null); offset += 1; // Advance to next starting offset if (maxBytes > 0 && offset > maxBytes) { break; } - bufreloff += 1; - if (bufreloff == firstbuf.length) { // If starting offset no longer falls in firstbuf - byte[] tmp = firstbuf; // Switch firstbuf with secondbuf - firstbuf = secondbuf; - secondbuf = tmp; - ra = in.read(secondbuf); // refill secondbuf (old firstbuf) with new bytes - if (monitor!=null) { - if (monitor.isCancelled()) return; + bufRelativeOffset += 1; + if (bufRelativeOffset == firstBuf.length) { // If starting offset no longer falls in firstbuf + byte[] tmp = firstBuf; // Switch firstbuf with secondbuf + firstBuf = secondBuf; + secondBuf = tmp; + ra = in.read(secondBuf); // refill secondbuf (old firstbuf) with new bytes + if (monitor != null) { + if (monitor.isCancelled()) { + return; + } monitor.setProgress(offset); } - if (ra != secondbuf.length) { - fullbuffers = 1; - if (ra < 0) + if (ra != secondBuf.length) { + fullBuffers = 1; + if (ra < 0) { ra = 0; + } tmp = new byte[ra]; - for(int i=0;i= 0 && (maxBytes <= 0 || offset < maxBytes)) { - if (secondbuf.length == 0) - fullbuffers = 0; - curstate = this; - subindex = bufreloff; - curbuf = firstbuf; + + while (fullBuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) { + if (secondBuf.length == 0) { + fullBuffers = 0; + } + curState = this; + subIndex = bufRelativeOffset; + curBuf = firstBuf; do { - if (curstate.success != null) - curstate.exportSuccess(match, offset); - if (subindex >= curbuf.length) { - if (curbuf == secondbuf) break; // Out of data, all pending patterns fail - curbuf = secondbuf; - subindex = 0; - if (curbuf.length==0) break; + if (curState.success != null) { + curState.exportSuccess(match, offset); } - curstate = curstate.trans[ 0xff & curbuf[subindex] ]; - subindex += 1; - } while(curstate != null); + if (subIndex >= curBuf.length) { + if (curBuf == secondBuf) { + break; // Out of data, all pending patterns fail + } + curBuf = secondBuf; + subIndex = 0; + if (curBuf.length == 0) { + break; + } + } + curState = curState.trans[0xff & curBuf[subIndex]]; + subIndex += 1; + } + while (curState != null); offset += 1; - bufreloff += 1; - if (bufreloff == firstbuf.length) { - if (fullbuffers == 0) break; - firstbuf = secondbuf; - fullbuffers = 0; - bufreloff = 0; - secondbuf = new byte[0]; + bufRelativeOffset += 1; + if (bufRelativeOffset == firstBuf.length) { + if (fullBuffers == 0) { + break; + } + firstBuf = secondBuf; + fullBuffers = 0; + bufRelativeOffset = 0; + secondBuf = new byte[0]; } } } - - static public ArrayList buildTransitionLevel(ArrayList prev,int pos) { + + static public ArrayList buildTransitionLevel( + ArrayList prev, int pos) { ArrayList res = new ArrayList(); Iterator iterator = prev.iterator(); - while(iterator.hasNext()) { // For each current state + while (iterator.hasNext()) { // For each current state SequenceSearchState next = iterator.next(); next.trans = new SequenceSearchState[256]; - for(int i=0;i<256;++i) { // Try every byte transition + for (int i = 0; i < 256; ++i) { // Try every byte transition next.buildSingleTransition(res, pos, i); } } - if (res.isEmpty()) return res; + if (res.isEmpty()) { + return res; + } // Prepare to dedup the states Collections.sort(res); ArrayList finalres = new ArrayList(); Iterator iter = res.iterator(); SequenceSearchState curpat = iter.next(); finalres.add(curpat); - while(iter.hasNext()) { + while (iter.hasNext()) { SequenceSearchState nextpat = iter.next(); int comp = curpat.compareTo(nextpat); if (comp == 0) { // Identical states @@ -364,12 +406,13 @@ public class SequenceSearchState implements Comparable { } return finalres; } - - static public SequenceSearchState buildStateMachine(ArrayList patterns) { + + static public SequenceSearchState buildStateMachine( + ArrayList patterns) { SequenceSearchState root = new SequenceSearchState(null); - for(int i=0;i { int level = 0; do { statelevel = buildTransitionLevel(statelevel, level); - level += 1; - } while(!statelevel.isEmpty()); + level += 1; + } + while (!statelevel.isEmpty()); return root; } } diff --git a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java index 283dd1cc65..631fce5623 100644 --- a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java +++ b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java @@ -141,7 +141,8 @@ public abstract class AbstractCreateDataBackgroundCmd { private static final String RTTI_4_NAME = "RTTI Complete Object Locator"; private List vfTableBlocks; + private List
rtti4Locations; + + private Address currentProcessingAddress; /** * Constructs a command for applying an RTTI4 dataType at an address. @@ -54,12 +59,44 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd(); + rtti4Locations.add(address); + } + + public CreateRtti4BackgroundCmd(List
addresses, List vfTableBlocks, + DataValidationOptions validationOptions, DataApplyOptions applyOptions) { + + super(Rtti4Model.DATA_TYPE_NAME, addresses.get(0), 1, validationOptions, applyOptions); + this.rtti4Locations = addresses; + this.vfTableBlocks = vfTableBlocks; + } + + @Override + protected boolean doApplyTo(Program program, TaskMonitor taskMonitor) + throws CancelledException { + + // process each potential RTTI4 entry, keeping track of good RTTI4 tables + List
goodRtti4Locations = new ArrayList
(); + boolean succeeded = false; + for (Address addr : rtti4Locations) { + currentProcessingAddress = addr; + succeeded |= super.doApplyTo(program, taskMonitor); + goodRtti4Locations.add(addr); + } + + // if any succeeded and should create associated data, make the vftables all at one time + if (succeeded && applyOptions.shouldFollowData()) { + createaAssociatedVfTables(program, goodRtti4Locations, taskMonitor); + } + + return succeeded; } @Override protected Rtti4Model createModel(Program program) { - if (model == null || program != model.getProgram()) { - model = new Rtti4Model(program, address, validationOptions); + if (model == null || program != model.getProgram() || + !getDataAddress().equals(model.getAddress())) { + model = new Rtti4Model(program, getDataAddress(), validationOptions); } return model; } @@ -75,7 +112,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd goodRtti4Locations, + TaskMonitor taskMonitor) throws CancelledException { - monitor.checkCanceled(); + MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables"); - Program program = model.getProgram(); + HashMap foundVFtables = new HashMap<>(); - Address rtti4Address = address; - int defaultPointerSize = program.getDefaultPointerSize(); - int alignment = defaultPointerSize; // Align the vf table based on the size of the pointers in it. - Set
directRtti4Refs = - ProgramMemoryUtil.findDirectReferences(program, vfTableBlocks, alignment, rtti4Address, - monitor); + for (Address rtti4Address : goodRtti4Locations) { - VfTableModel validVfTableModel = null; - for (Address possibleVfMetaAddr : directRtti4Refs) { + byte[] bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti4Address); + addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes); + + bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti4Address); + + addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes); + } + + AddressSet searchSet = new AddressSet(); + for (MemoryBlock block : vfTableBlocks) { + searchSet.add(block.getStart(), block.getEnd()); + } + + searcher.search(program, searchSet, monitor); + + // did the search, now process the results + boolean didSome = false; + for (Address rtti4Address : goodRtti4Locations) { monitor.checkCanceled(); - Address possibleVfTableAddr = possibleVfMetaAddr.add(defaultPointerSize); + VfTableModel vfTableModel = foundVFtables.get(rtti4Address); + if (vfTableModel == null) { + String message = + "No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address; + handleErrorMessage(program, rtti4Address, message); + continue; + } - // Validate the model. Don't apply the command if invalid. - try { - VfTableModel vfTableModel = - new VfTableModel(program, possibleVfTableAddr, validationOptions); - vfTableModel.validate(); + CreateVfTableBackgroundCmd cmd = + new CreateVfTableBackgroundCmd(vfTableModel, applyOptions); + didSome |= cmd.applyTo(program, monitor); + } - if (validVfTableModel != null) { - String message = "More than one possible vfTable found for " + - Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address; - handleErrorMessage(program, rtti4Address, message); - return false; + return didSome; + } + + /** + * Add a search pattern, to the searcher, for the set of bytes representing an rtti4 location. + * Only one VFTable for is allowed for an RTT4 location, last one in wins and gets created. + * + * @param searcher byte pattern searcher + * @param foundVFtables list of addresses accumulated when actual search is performed + * @param rtti4Address location of rttiAddress to find vfTable for + * @param bytes bytes representing rtti4Addres to be found in memory + */ + private void addByteSearchPattern(MemoryBytePatternSearcher searcher, + HashMap foundVFtables, Address rtti4Address, byte[] bytes) { + + if (bytes == null) { + return; + } + + GenericMatchAction
action = new GenericMatchAction
(rtti4Address) { + @Override + public void apply(Program prog, Address addr, Match match) { + + Address possibleVfTableAddr = addr.add(prog.getDefaultPointerSize()); + + // See if VfTable is valid, and add to rtti4 to vfTable map + try { + VfTableModel vfTableModel = + new VfTableModel(prog, possibleVfTableAddr, validationOptions); + vfTableModel.validate(); + + VfTableModel existing = foundVFtables.put(rtti4Address, vfTableModel); + + if (existing != null) { + // potential table already found, is an error, don't know which is right + String message = "More than one possible vfTable found for " + + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address; + handleErrorMessage(prog, rtti4Address, message); + } + } + catch (InvalidDataTypeException e) { + // This isn't a valid model. } - validVfTableModel = vfTableModel; } - catch (InvalidDataTypeException e) { - continue; // This isn't a valid model. - } - } + }; - if (validVfTableModel == null) { - String message = - "No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address; - handleErrorMessage(program, rtti4Address, message); - return false; - } + GenericByteSequencePattern
genericByteMatchPattern = + new GenericByteSequencePattern
(bytes, action); - monitor.checkCanceled(); - - CreateVfTableBackgroundCmd cmd = - new CreateVfTableBackgroundCmd(validVfTableModel, applyOptions); - return cmd.applyTo(program, monitor); + searcher.addPattern(genericByteMatchPattern); } @Override @@ -178,19 +255,31 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd 0) { DataTypeManager dataTypeManager = program.getDataTypeManager(); PointerDataType pointerDt = new PointerDataType(dataTypeManager); diff --git a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java index 9e3f60485e..ac20da8f0a 100644 --- a/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java +++ b/Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java @@ -30,6 +30,7 @@ import ghidra.program.model.lang.UndefinedValueException; import ghidra.program.model.listing.Program; import ghidra.program.model.mem.MemoryBlock; import ghidra.program.util.ProgramMemoryUtil; +import ghidra.util.bytesearch.*; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; @@ -41,7 +42,7 @@ public class RttiAnalyzer extends AbstractAnalyzer { private static final String NAME = "Windows x86 PE RTTI Analyzer"; private static final String DESCRIPTION = - "This analyzer finds and creates all of the RTTI metadata structures and their associated vf tables."; + "Finds and creates RTTI metadata structures and associated vf tables."; // TODO If we want the RTTI analyzer to find all type descriptors regardless of whether // they are used for RTTI, then change the CLASS_PREFIX_CHARS to ".". Need to be @@ -133,6 +134,7 @@ public class RttiAnalyzer extends AbstractAnalyzer { monitor.setMaximum(possibleRtti0Addresses.size()); monitor.setMessage("Creating RTTI Data..."); + ArrayList
rtti0Locations = new ArrayList
(); int count = 0; for (Address rtti0Address : possibleRtti0Addresses) { monitor.checkCanceled(); @@ -157,29 +159,29 @@ public class RttiAnalyzer extends AbstractAnalyzer { rtti0Address, validationOptions, applyOptions); typeDescCmd.applyTo(program, monitor); - // Create any valid RTTI4s for this TypeDescriptor - processRtti4sForRtti0(program, rtti0Address, monitor); + rtti0Locations.add(rtti0Address); } + + // Create any valid RTTI4s for this TypeDescriptor + processRtti4sForRtti0(program, rtti0Locations, monitor); } - private void processRtti4sForRtti0(Program program, Address rtti0Address, TaskMonitor monitor) - throws CancelledException { + private void processRtti4sForRtti0(Program program, List
rtti0Locations, + TaskMonitor monitor) throws CancelledException { - List rDataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program, + List dataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program, program.getMemory(), ".rdata", monitor); + dataBlocks.addAll(ProgramMemoryUtil.getMemoryBlocksStartingWithName(program, + program.getMemory(), ".data", monitor)); + List
rtti4Addresses = - getRtti4Addresses(program, rDataBlocks, rtti0Address, validationOptions, monitor); + getRtti4Addresses(program, dataBlocks, rtti0Locations, validationOptions, monitor); - for (Address rtti4Address : rtti4Addresses) { - - monitor.checkCanceled(); - - CreateRtti4BackgroundCmd cmd = - new CreateRtti4BackgroundCmd(rtti4Address, rDataBlocks, validationOptions, - applyOptions); - cmd.applyTo(program, monitor); - } + // create all found RTTI4 tables at once + CreateRtti4BackgroundCmd cmd = new CreateRtti4BackgroundCmd(rtti4Addresses, dataBlocks, + validationOptions, applyOptions); + cmd.applyTo(program, monitor); } /** @@ -187,67 +189,134 @@ public class RttiAnalyzer extends AbstractAnalyzer { * the RTTI 0 at the indicated base address. * @param program the program containing the RTTI 0 data structure. * @param rtti4Blocks the memory blocks to be searched for RTTI4 structures. - * @param rtti0Address the base address of the RTTI 0 structure in the program + * @param rtti0Locations the base addresses of the RTTI 0 structure in the program * @param validationOptions options indicating how validation is performed for data structures - * @param monitor the task monitor for cancelling a task + * @param monitor the task monitor for canceling a task * @return the RTTI 4 base addresses associated with the RTTI 0 * @throws CancelledException if the user cancels this task. */ private static List
getRtti4Addresses(Program program, List rtti4Blocks, - Address rtti0Address, DataValidationOptions validationOptions, TaskMonitor monitor) - throws CancelledException { + List
rtti0Locations, DataValidationOptions validationOptions, + TaskMonitor monitor) throws CancelledException { monitor.checkCanceled(); - List
addresses = new ArrayList<>(); // the RTTI 4 addresses - int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset(); - Set
refsToRtti0 = getRefsToRtti0(program, rtti4Blocks, rtti0Address); + List
addresses = + getRefsToRtti0(program, rtti4Blocks, rtti0Locations, validationOptions, monitor); - // for each RTTI 0 now see if we can get RTTI4s that refer to it. - for (Address refAddress : refsToRtti0) { - - monitor.checkCanceled(); - - Address possibleRtti4Address; - try { - possibleRtti4Address = refAddress.subtractNoWrap(rtti0PointerOffset); - } - catch (AddressOverflowException e) { - continue; // Couldn't get an Rtti4 address. - } - - Rtti4Model rtti4Model = - new Rtti4Model(program, possibleRtti4Address, validationOptions); - try { - rtti4Model.validate(); - } - catch (InvalidDataTypeException e) { - continue; // Only process valid RTTI 4 data. - } - - // Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly - // through the RTTI 3. - boolean refersToRtti0 = rtti4Model.refersToRtti0(rtti0Address); - if (!refersToRtti0) { - continue; // Only process valid RTTI 4 data. - } - - addresses.add(possibleRtti4Address); - } return addresses; } - private static Set
getRefsToRtti0(Program program, List dataBlocks, - Address rtti0Address) throws CancelledException { - Set
refsToRtti0; - if (MSDataTypeUtils.is64Bit(program)) { - refsToRtti0 = ProgramMemoryUtil.findImageBaseOffsets32(program, 4, rtti0Address, - TaskMonitor.DUMMY); + /** For each of the RTTI0 locations found locate the associated RTTI4 structure referring to it. + * + * @param program program to be searched + * @param dataBlocks dataBlocks to search + * @param rtti0Locations list of known rtti0 locations + * @param validationOptions options for validation of found RTTI4 entries + * @param monitor to cancel + * @return list of found RTTI4 references to known RTTI0 locations + * @throws CancelledException if canceled + */ + private static List
getRefsToRtti0(Program program, List dataBlocks, + List
rtti0Locations, DataValidationOptions validationOptions, + TaskMonitor monitor) throws CancelledException { + + List
addresses = new ArrayList<>(); // the RTTI 4 addresses + + int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset(); + + MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI0 refernces"); + + for (Address rtti0Address : rtti0Locations) { + byte[] bytes; + if (MSDataTypeUtils.is64Bit(program)) { + // 64-bit programs will have the addresses as offsets from the image base (BOS) + bytes = ProgramMemoryUtil.getImageBaseOffsets32Bytes(program, 4, rtti0Address); + + addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset, + rtti0Address, bytes); + } + else { + // 32-bit could have direct address in memory + bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti0Address); + + addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset, + rtti0Address, bytes); + + // or referenced by a shifted pointer based on program data organization shift amount + bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti0Address); + + addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset, + rtti0Address, bytes); + } } - else { - refsToRtti0 = ProgramMemoryUtil.findDirectReferences(program, dataBlocks, - program.getDefaultPointerSize(), rtti0Address, TaskMonitor.DUMMY); + + AddressSet searchSet = new AddressSet(); + for (MemoryBlock block : dataBlocks) { + searchSet.add(block.getStart(), block.getEnd()); } - return refsToRtti0; + + searcher.search(program, searchSet, monitor); + + return addresses; + } + + /** + * Add a search pattern, to the searcher, for the set of bytes representing an address + * + * @param searcher pattern searcher + * @param validationOptions RTTI4 validation options + * @param addresses list of found valid RTTI4 locations accumulated during actual search + * @param rtti0PointerOffset offset of pointer in RTTI4 entry to RTTI0 + * @param rtti0Address RTTI0 address corresponding to pattern of bytes + * @param bytes pattern of bytes in memory corresponding to address + */ + private static void addByteSearchPattern(MemoryBytePatternSearcher searcher, + DataValidationOptions validationOptions, List
addresses, + int rtti0PointerOffset, Address rtti0Address, byte[] bytes) { + + // no pattern bytes. + if (bytes == null) { + return; + } + + // Each time a match for this byte pattern validate as an RTTI4 and add to list + GenericMatchAction
action = new GenericMatchAction
(rtti0Address) { + @Override + public void apply(Program prog, Address addr, Match match) { + Address possibleRtti4Address; + try { + possibleRtti4Address = addr.subtractNoWrap(rtti0PointerOffset); + } + catch (AddressOverflowException e) { + return; // Couldn't get an Rtti4 address. + } + + Rtti4Model rtti4Model = + new Rtti4Model(prog, possibleRtti4Address, validationOptions); + try { + rtti4Model.validate(); + } + catch (InvalidDataTypeException e) { + return; // Only process valid RTTI 4 data. + } + + // Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly + // through the RTTI 3. + boolean refersToRtti0 = rtti4Model.refersToRtti0(getMatchValue()); + if (!refersToRtti0) { + return; // Only process valid RTTI 4 data. + } + + // add to list of RTTI4 locations to be processed later + addresses.add(possibleRtti4Address); + } + }; + + // create a Pattern of the bytes and the MatchAction to perform upon a match + GenericByteSequencePattern
genericByteMatchPattern = + new GenericByteSequencePattern
(bytes, action); + + searcher.addPattern(genericByteMatchPattern); } }