yetangitu/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2025-10-05 02:39:44 +02:00
Code Issues Releases Wiki Activity

GT-3341_emteere_RTTIByteSearchPerformance speed up multiple byte pattern

Browse source 
search, comment
This commit is contained in:
emteere 2020-02-06 22:28:42 -05:00
parent 193f6f40e8
commit b426f065f5
30 changed files with 1162 additions and 462 deletions
Download patch file Download diff file Expand all files Collapse all files

110
Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java
View file

@ -15,7 +15,8 @@
*/
package ghidra.app.plugin.core.analysis;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.cmd.data.CreateDataCmd;
import ghidra.app.services.*;
@ -23,15 +24,17 @@ import ghidra.app.util.importer.MessageLog;
import ghidra.framework.options.Options;
import ghidra.program.model.address.*;
import ghidra.program.model.data.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.listing.BookmarkType;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.Memory;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
private static final String NAME = "Embedded Media";
private static final String DESCRIPTION =
"Finds and tries to apply embedded media data types (ie png, gif, jpeg, wav) in current program.";
"Finds embedded media data types (ie png, gif, jpeg, wav)";
private static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks";
private static final String OPTION_DESCRIPTION_CREATE_BOOKMARKS =
@ -52,71 +55,62 @@ public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
throws CancelledException {
Memory memory = program.getMemory();
AddressSetView initializedAddressSet = memory.getLoadedAndInitializedAddressSet();
AddressSet initialedSearchSet = set.intersect(initializedAddressSet);
AddressSetView validMemorySet = memory.getLoadedAndInitializedAddressSet();
AddressSet searchSet = set.intersect(validMemorySet);
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("Embedded Media");
List<Address> foundMedia = new ArrayList<>();
foundMedia = scanForMedia(program, new GifDataType(), "GIF 87", GifDataType.MAGIC_87,
GifDataType.GIFMASK, initialedSearchSet, memory, monitor);
addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 87",
GifDataType.MAGIC_87, GifDataType.GIFMASK);
foundMedia.addAll(scanForMedia(program, new GifDataType(), "GIF 89", GifDataType.MAGIC_89,
GifDataType.GIFMASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 89",
GifDataType.MAGIC_89, GifDataType.GIFMASK);
foundMedia.addAll(scanForMedia(program, new PngDataType(), "PNG", PngDataType.MAGIC,
PngDataType.MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new PngDataType(), "PNG",
PngDataType.MAGIC, PngDataType.MASK);
foundMedia.addAll(scanForMedia(program, new JPEGDataType(), "JPEG", JPEGDataType.MAGIC,
JPEGDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new JPEGDataType(), "JPEG",
JPEGDataType.MAGIC, JPEGDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new WAVEDataType(), "WAVE", WAVEDataType.MAGIC,
WAVEDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new WAVEDataType(), "WAVE",
WAVEDataType.MAGIC, WAVEDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new AUDataType(), "AU", AUDataType.MAGIC,
AUDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new AUDataType(), "AU",
AUDataType.MAGIC, AUDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new AIFFDataType(), "AIFF", AIFFDataType.MAGIC,
AIFFDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new AIFFDataType(), "AIFF",
AIFFDataType.MAGIC, AIFFDataType.MAGIC_MASK);
return true;
searcher.search(program, searchSet, monitor);
return foundMedia.size() > 0;
}
private List<Address> scanForMedia(Program program, DataType dt, String mediaName,
byte[] mediaBytes, byte[] mask, AddressSetView addresses, Memory memory,
TaskMonitor monitor) {
monitor.setMessage("Scanning for " + mediaName + " Embedded Media");
monitor.initialize(addresses.getNumAddresses());
List<Address> foundMediaAddresses = new ArrayList<>();
Iterator<AddressRange> iterator = addresses.iterator();
while (iterator.hasNext()) {
if (monitor.isCancelled()) {
return foundMediaAddresses;
private void addByteSearchPattern(MemoryBytePatternSearcher searcher, Program program,
List<Address> foundMedia, DataType mediaDT, String mediaName, byte[] bytes,
byte[] mask) {
if (bytes == null) {
return;
}
AddressRange range = iterator.next();
Address start = range.getMinAddress();
Address end = range.getMaxAddress();
Address found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
while (found != null && !monitor.isCancelled()) {
GenericMatchAction<DataType> action = new GenericMatchAction<DataType>(mediaDT) {
@Override
public void apply(Program prog, Address addr, Match match) {
//See if it is already an applied media data type
Data data = program.getListing().getDefinedDataAt(found);
int skipLen = 1;
if (data == null) {
if (!program.getListing().isUndefined(addr, addr)) {
return;
}
try {
CreateDataCmd cmd = new CreateDataCmd(found, dt);
CreateDataCmd cmd = new CreateDataCmd(addr, mediaDT);
if (cmd.applyTo(program)) {
if (createBookmarksEnabled) {
program.getBookmarkManager().setBookmark(found,
BookmarkType.ANALYSIS, "Embedded Media",
"Found " + mediaName + " Embedded Media");
program.getBookmarkManager().setBookmark(addr, BookmarkType.ANALYSIS,
"Embedded Media", "Found " + mediaName + " Embedded Media");
}
foundMediaAddresses.add(found);
//have to get the actual applied data to find the actual length to skip because until then it can't compute the length due to the data type being dynamic
skipLen = program.getListing().getDataAt(found).getLength();
foundMedia.add(addr);
}
}
//If media does not apply correctly then it is not really a that media data type or there is other data in the way
@ -124,20 +118,12 @@ public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
// Not a valid embedded media or no room to apply it so just ignore it and skip it
}
}
// skip either the valid data that was found or skip one byte
// then do the next search
try {
start = found.add(skipLen);
found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
}
catch (AddressOutOfBoundsException e) {
// If media was at the very end of the address space, we will end up here
break;
}
}
}
};
return foundMediaAddresses;
GenericByteSequencePattern<DataType> genericByteMatchPattern =
new GenericByteSequencePattern<DataType>(bytes, mask, action);
searcher.addPattern(genericByteMatchPattern);
}
@Override

86
Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java
View file

@ -177,8 +177,7 @@ public class ProgramMemoryUtil {
MemoryBlock[] tmpBlocks = new MemoryBlock[blocks.length];
int j = 0;
for (MemoryBlock block : blocks) {
if ((block.isInitialized() && withBytes) ||
(!block.isInitialized() && !withBytes)) {
if ((block.isInitialized() && withBytes) || (!block.isInitialized() && !withBytes)) {
tmpBlocks[j++] = block;
}
}
@ -493,6 +492,30 @@ public class ProgramMemoryUtil {
monitor = TaskMonitorAdapter.DUMMY_MONITOR;
}
byte[] addressBytes = getDirectAddressBytes(program, toAddress);
byte[] shiftedAddressBytes = getShiftedDirectAddressBytes(program, toAddress);
Memory memory = program.getMemory();
Set<Address> dirRefsAddrs = new TreeSet<>();
findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
}
return dirRefsAddrs;
}
/**
* Get a representation of an address as it would appear in bytes in memory.
*
* @param program program
* @param toAddress target address
* @return byte representation of toAddress
*/
public static byte[] getDirectAddressBytes(Program program, Address toAddress) {
Memory memory = program.getMemory();
boolean isBigEndian = memory.isBigEndian();
@ -536,6 +559,31 @@ public class ProgramMemoryUtil {
addressBytes, 0, addressBytes.length);
}
return addressBytes;
}
/**
* returns shifted address bytes if they are different than un-shifted
*
* @param program program
* @param toAddress target address
* @return shifted bytes, null if same as un-shifted
*/
public static byte[] getShiftedDirectAddressBytes(Program program, Address toAddress) {
byte[] addressBytes = getDirectAddressBytes(program, toAddress);
Memory memory = program.getMemory();
boolean isBigEndian = memory.isBigEndian();
DataConverter dataConverter;
if (isBigEndian) {
dataConverter = new BigEndianDataConverter();
}
else {
dataConverter = new LittleEndianDataConverter();
}
byte[] shiftedAddressBytes = null;
DataTypeManager dataTypeManager = program.getDataTypeManager();
DataOrganization dataOrganization = dataTypeManager.getDataOrganization();
@ -554,24 +602,23 @@ public class ProgramMemoryUtil {
}
}
// don't need this anymore - finding all 16 bit addrs in whole prog
// AddressRange segmentRange = null;
// if (toAddress instanceof SegmentedAddress) {
// // Restrict search to currentSegment range
// SegmentedAddressSpace segSpace = (SegmentedAddressSpace) toAddress.getAddressSpace();
// segmentRange =
// new AddressRangeImpl(segSpace.getAddress(currentSegment, 0), segSpace.getAddress(
// currentSegment, 0xffff));
// }
Set<Address> dirRefsAddrs = new TreeSet<>();
findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
return shiftedAddressBytes;
}
return dirRefsAddrs;
public static byte[] getImageBaseOffsets32Bytes(Program program, int alignment,
Address toAddress) {
Address imageBase = program.getImageBase();
long offsetValue = toAddress.subtract(imageBase);
int offsetSize = 4; // 32 bit offset
byte[] bytes = new byte[offsetSize];
for (int i = 0; i < offsetSize; i++) {
bytes[i] = (byte) offsetValue;
offsetValue >>= 8; // Shift by a single byte.
}
return bytes;
}
/**
@ -627,8 +674,7 @@ public class ProgramMemoryUtil {
if (!block.isInitialized()) {
continue;
}
if (memoryRange != null &&
!memoryRange.intersects(block.getStart(), block.getEnd())) {
if (memoryRange != null && !memoryRange.intersects(block.getStart(), block.getEnd())) {
// skip blocks which do not correspond to currentSeg
continue;
}

21
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java
View file

@ -19,12 +19,31 @@ import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* ByteSearch post search rule when a pattern is found, The pattern must have a certain
* alignment at an offset from the location the pattern matches. The alignment is
* specified by the mask bits that must be zero.
*
* mark is the offset in bytes from the start of the matching pattern.
*
* align 2 = 0x1 - lower bit must be zero
* align 4 = 0x3 - lower two bits must be zero
* align 8 = 0x7 - lower three bits must be zero
* align 16 = 0xF - lower four bits must be zero
* ....
* Other strange alignments could be specified, but most likely the above suffice.
*
* The pattern can be constructed or restored from XML of the form:
*
* <align mark="0" bits="1"/>
*
*/
public class AlignRule implements PostRule {
private int mark; // Position, relative to start of pattern, to check alignment at
private int alignmask; // Mask of bits that must be zero
public AlignRule() {
}

118
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java
View file

@ -21,6 +21,19 @@ import java.util.zip.CRC32;
import ghidra.xml.XmlPullParser;
/**
* A pattern of bits/mask to match to a stream of bytes. The bits/mask can be of any length.
* The sequence can be initialized by:
*
* a string
* an array of bytes (no mask)
* an array of bytes and for mask
*
* The dits represent bits(binary) or nibbles(hex) that are don't care, for example:
* 0x..d.4de2 ....0000 .1...... 00101101 11101001
* where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
*/
public class DittedBitSequence {
//Given a byte 0-255 (NOT a signed byte), retrieves its popcount.
@ -42,7 +55,7 @@ public class DittedBitSequence {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 //240-255
};
protected int index; // Unique index assigned to this sequence
private int index; // Unique index assigned to this sequence
private byte[] bits; // value bits contained in the sequence
private byte[] dits; // a 1 indicates the bit is not ditted
@ -53,7 +66,9 @@ public class DittedBitSequence {
/**
* Constructor from a ditted-bit-sequence string where white space is ignored (e.g., "10..11.0");
* @param dittedBitData
*
* @param dittedBitData ditted sequence specified as a string
*
* @throws IllegalArgumentException if invalid dittedBitData specified
*/
public DittedBitSequence(String dittedBitData) {
@ -64,8 +79,8 @@ public class DittedBitSequence {
* Constructor from a ditted-bit string where white space is ignored. If there are no dits,
* {@code hex} is true, and {@code hex} does not begin with {code 0x}, {@code 0x} will be
* prepended to the string before constructing the {@link DittedBitSequence}.
* @param dittedBitData
* @param hex
* @param dittedBitData string of bits and dits or hex numbers and dits (e.g., 0.1..0, 0xAB..)
* @param hex true to force hex on the sequence
*/
public DittedBitSequence(String dittedBitData, boolean hex) {
if (hex && !dittedBitData.contains(".")) {
@ -101,7 +116,8 @@ public class DittedBitSequence {
/**
* Construct a sequence of bytes to search for. No bits are masked off.
* @param bytes
*
* @param bytes byte values that must match
*/
public DittedBitSequence(byte[] bytes) {
bits = bytes;
@ -164,22 +180,40 @@ public class DittedBitSequence {
return true;
}
public DittedBitSequence concatenate(DittedBitSequence op2) {
/**
* Concatenates a sequence to the end of another sequence and
* returns a new sequence.
*
* @param toConat sequence to concatenate to this sequence
*
* @return a new sequence that is the concat of this and toConcat
*/
public DittedBitSequence concatenate(DittedBitSequence toConat) {
DittedBitSequence res = new DittedBitSequence();
res.bits = new byte[bits.length + op2.bits.length];
res.bits = new byte[bits.length + toConat.bits.length];
res.dits = new byte[res.bits.length];
for (int i = 0; i < bits.length; ++i) {
res.bits[i] = bits[i];
res.dits[i] = dits[i];
}
for (int i = 0; i < op2.bits.length; ++i) {
res.bits[bits.length + i] = op2.bits[i];
res.dits[bits.length + i] = op2.dits[i];
for (int i = 0; i < toConat.bits.length; ++i) {
res.bits[bits.length + i] = toConat.bits[i];
res.dits[bits.length + i] = toConat.dits[i];
}
return res;
}
/**
* Check for a match of a value at a certain offset in the pattern.
* An outside matcher will keep track of the match position within this
* ditted bit sequence. Then call this method to match.
*
* @param pos position in the pattern to match
* @param val a byte to be match at the given byte offset in the pattern
*
* @return true if the byte matches the sequence mask/value
*/
public boolean isMatch(int pos, int val) {
if (pos >= bits.length) {
return false;
@ -187,14 +221,38 @@ public class DittedBitSequence {
return ((byte) (val & dits[pos])) == bits[pos];
}
/**
* Set a an index in a larger sequence, or identifing id on this pattern
*
* @param index - index in match sequence, or unique id
*/
public void setIndex(int index) {
this.index = index;
}
/**
* Get the index or identifying id attached to this pattern
*
* @return index or unique id attached to this sequence
*/
public int getIndex() {
return index;
}
/**
* get the size of this sequence in bytes
*
* @return size in bytes
*/
public int getSize() {
return bits.length;
}
/**
* Get number of bits that must be 0/1
*
* @return number of bits that are not don't care (ditted)
*/
public int getNumFixedBits() {
int popcnt = 0;
for (byte dit : dits) {
@ -203,7 +261,11 @@ public class DittedBitSequence {
return popcnt;
}
//Return the number of dits.
/**
* Get number of bits that are ditted (don't care)
*
* @return number of ditted bits (don't care)
*/
public int getNumUncertainBits() {
int popcnt = 0;
for (byte dit : dits) {
@ -235,6 +297,11 @@ public class DittedBitSequence {
return buf.toString();
}
/**
* get a ditted hex string representing this sequence
*
* @return ditted hex string
*/
public String getHexString() {
String uncompressed = this.toString();
String[] parts = uncompressed.trim().split(" ");
@ -260,6 +327,17 @@ public class DittedBitSequence {
return sb.toString();
}
/**
* restore ditted string from XML stream with hex/binary ditted sequences in the form:
* <data> 0x..d.4de2 ....0000 .1...... 00101101 11101001 </data>
* where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
*
* @param parser XML pull parser stream
*
* @return number of bytes read from XML <data> tag
*
* @throws IOException if XML read has an error
*/
protected int restoreXmlData(XmlPullParser parser) throws IOException {
parser.start("data");
String text = parser.end().getText();
@ -272,6 +350,16 @@ public class DittedBitSequence {
}
}
/**
* Initialize this sequence with a ditted sequence from a string in the form
* (e.g. - 011...1., 0x.F, 01110011 0xAB)
*
* @param text ditted sequence
*
* @return number of bytes in the ditted sequence
*
* @throws IllegalArgumentException if string is malformed
*/
private int initFromDittedStringData(String text) throws IllegalArgumentException {
int markOffset = -1;
int mode = -1; // -1: looking for start, -2: skip to EOL, 0: hex mode, 1: binary mode
@ -372,6 +460,13 @@ public class DittedBitSequence {
return markOffset;
}
/**
* Get the number of bits that are fixed, not ditted (don't care)
*
* @param marked number of bytes in the pattern to check
*
* @return number of initial fixed bits
*/
public int getNumInitialFixedBits(int marked) {
if (dits == null) {
return 0;
@ -385,5 +480,4 @@ public class DittedBitSequence {
}
return popcnt;
}
}

5
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,6 +19,9 @@ import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Program;
import ghidra.xml.XmlPullParser;
/**
* Dummy action attached to a match sequence. Action is not restored from XML
*/
public class DummyMatchAction implements MatchAction {
@Override
@ -32,4 +34,3 @@ public class DummyMatchAction implements MatchAction {
}
}

46
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java Normal file
View file

@ -0,0 +1,46 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import ghidra.program.model.data.DataType;
/**
* Templated simple DittedBitSequence Pattern for a byte/mask pattern and associated action.
* The DittedBitSequence is provided by value and mask in byte arrays.
*
* This class is normally used to find some number of SequencePatterns within a seqence of bytes.
* When the byte/mask pattern is matched, the GenericMatchAction will be "applied".
*
* @param <T> the class of match action, used to specify a specialized momento to be used by the action when it is "applied".
*/
public class GenericByteSequencePattern<T> extends Pattern {
public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction<T> action) {
super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]);
MatchAction[] matchActions = getMatchActions();
matchActions[0] = action;
}
public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask,
GenericMatchAction<DataType> action) {
super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]);
MatchAction[] matchActions = getMatchActions();
matchActions[0] = action;
}
}

42
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java Normal file
View file

@ -0,0 +1,42 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
/**
* Template for generic match action attached to a match sequence.
* Used to store an associated value to the matching sequence.
* The associated value can be retrieved when the sequence is matched.
*
* @param <T> - object to attach to match sequence, generally used to specify
* a specialized momento to be used by the action when it is "applied".
*/
public class GenericMatchAction<T> extends DummyMatchAction {
T matchValue;
/*
* construct with an appropriate object that can be used when the action is applied
*/
public GenericMatchAction(T matchValue) {
this.matchValue = matchValue;
}
/**
* @return the specialized object associated with this match action
*/
public T getMatchValue() {
return this.matchValue;
}
}

46
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Match.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java
View file

@ -15,13 +15,21 @@
*/
package ghidra.util.bytesearch;
/**
* Represents a match of a DittedBitSequence at a given offset in a byte sequence.
*
* There is a hidden assumption that the sequence is actually a Pattern
* that might have a ditted-bit-sequence, a set of match actions,
* and post match rules/checks
*
*/
public class Match {
private DittedBitSequence sequence; // Pattern that matches
private long offset; // starting offset within bytestream of match
public Match(DittedBitSequence seq, long off) {
sequence = seq;
offset = off;
public Match(DittedBitSequence sequence, long offset) {
this.sequence = sequence;
this.offset = offset;
}
/**
@ -39,42 +47,70 @@ public class Match {
return sequence.getNumFixedBits() - sequence.getNumInitialFixedBits(marked);
}
/**
* @return actions associated with this match
*/
public MatchAction[] getMatchActions() {
return ((Pattern) sequence).getMatchActions();
}
/**
* @return size in bytes of sequence
*/
public int getSequenceSize() {
return sequence.getSize();
}
/**
* @return index of sequence in a possibly longer set of sequences
*/
public int getSequenceIndex() {
return sequence.getIndex();
}
/**
* @return the offset of the match within a longer byte sequence
*/
public long getMarkOffset() {
return offset + ((Pattern) sequence).getMarkOffset();
}
/**
* @return offset of match in sequence of bytes
*/
public long getMatchStart() {
return offset;
}
/**
* Check that the possible post rules are satisfied
*
* @param streamoffset offset within from match location to check postrules.
*
* @return true if post rules are satisfied
*/
public boolean checkPostRules(long streamoffset) {
long curoffset = streamoffset + offset;
Pattern pattern = (Pattern) sequence;
PostRule[] postRules = pattern.getPostRules();
for (int i = 0; i < postRules.length; ++i) {
if (!postRules[i].apply(pattern, curoffset)) {
for (PostRule postRule : postRules) {
if (!postRule.apply(pattern, curoffset)) {
return false;
}
}
return true;
}
/**
* @return ditted bit sequence as a string
*/
public String getHexString() {
return sequence.getHexString();
}
/**
* @return the sequence that was matched
*/
public DittedBitSequence getSequence() {
return sequence;
}

4
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/MatchAction.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,8 +20,7 @@ import ghidra.program.model.listing.Program;
import ghidra.xml.XmlPullParser;
/**
* Action that should be applied to a Program at the Address a pattern matches
*
* Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern
*/
public interface MatchAction {
public void apply(Program program, Address addr, Match match);

179
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java Normal file
View file

@ -0,0 +1,179 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import ghidra.program.model.address.*;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* Multi pattern/mask/action memory searcher
*
* Preload search patterns and actions, then call search method.
*/
public class MemoryBytePatternSearcher {
private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;
ArrayList<Pattern> patternList;
private String searchName = "";
/**
* Create with pre-created patternList
*
* @param patternList - list of patterns(bytes/mask/action)
*/
public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {
this.searchName = searchName;
this.patternList = new ArrayList<Pattern>(patternList);
}
/**
* Create with no patternList, must add patterns before searching
*
*/
public MemoryBytePatternSearcher(String searchName) {
this.searchName = searchName;
patternList = new ArrayList<>();
}
/**
* Add a search pattern
* @param pattern - pattern(bytes/mask/action)
*/
public void addPattern(Pattern pattern) {
patternList.add(pattern);
}
/**
* Search initialized memory blocks for all patterns(bytes/mask/action).
* Call associated action for each pattern matched.
*
* @param program to be searched
* @param searchSet set of bytes to restrict search to
* @param monitor allow canceling and reporting of progress
*
* @throws CancelledException if canceled
*/
public void search(Program program, AddressSetView searchSet, TaskMonitor monitor)
throws CancelledException {
SequenceSearchState root = SequenceSearchState.buildStateMachine(patternList);
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block2 : blocks) {
MemoryBlock block = block2;
if (!searchSet.intersects(block.getStart(), block.getEnd())) {
continue;
}
try {
searchBlock(root, program, block, searchSet, monitor);
}
catch (IOException e) {
Msg.error(this, "Unable to scan block " + block.getName() + " for patterns");
}
}
}
/**
* Search through bytes of a memory block using the finite state machine -root-
* Apply any additional rules for matching patterns.
*
* @param program is the Program being searched
* @param block is the specific block of bytes being searched
*
* @throws IOException exception during read of memory
* @throws CancelledException canceled search
*/
private void searchBlock(SequenceSearchState rootState, Program program, MemoryBlock block,
AddressSetView restrictSet, TaskMonitor monitor)
throws IOException, CancelledException {
// if no restricted set, make restrict set the full block
AddressSet doneSet = new AddressSet(restrictSet);
if (doneSet.isEmpty()) {
doneSet.addRange(block.getStart(), block.getEnd());
}
doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
Address blockStartAddr = block.getStart();
// pull each range off the restricted set
AddressRangeIterator addressRanges = doneSet.getAddressRanges();
while (addressRanges.hasNext()) {
monitor.checkCanceled();
AddressRange addressRange = addressRanges.next();
monitor.setMessage(searchName + " Search");
monitor.initialize(doneSet.getNumAddresses());
monitor.setProgress(0);
ArrayList<Match> mymatches = new ArrayList<>();
long streamoffset = blockStartAddr.getOffset();
// Give block a starting/ending point before this address to search
// patterns might start before, since they have a pre-pattern
// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
// normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
// normally the whole memory blocks will get searched.
long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
if (blockOffset <= 0) {
// don't go before the block start
blockOffset = 0;
}
// compute number of bytes in the range + 1, and don't search more than that.
long maxBlockSearchLength =
addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
InputStream data = block.getData();
data.skip(blockOffset);
rootState.apply(data, maxBlockSearchLength, mymatches, monitor);
monitor.checkCanceled();
monitor.setMessage(searchName + " (Examine Matches)");
monitor.initialize(mymatches.size());
monitor.setProgress(0);
// TODO: DANGER there is much offset<-->address calculation here
// should be OK, since they are all relative to the block.
for (int i = 0; i < mymatches.size(); ++i) {
monitor.checkCanceled();
monitor.setProgress(i);
Match match = mymatches.get(i);
Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
if (!match.checkPostRules(streamoffset + blockOffset)) {
continue;
}
MatchAction[] matchactions = match.getMatchActions();
for (MatchAction matchaction : matchactions) {
matchaction.apply(program, addr, match);
}
}
}
}
}

10
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Pattern.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java
View file

@ -24,6 +24,13 @@ import generic.jar.ResourceFile;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.*;
/**
* Pattern is an association of a DittedBitSequence to match,
* a set of post rules after a match is found that must be satisfied,
* and a set of actions to be taken if the pattern matches.
*
* These patterns can be restored from an XML file.
*/
public class Pattern extends DittedBitSequence {
private int markOffset; // Within pattern what is the 'marked' byte
@ -37,7 +44,8 @@ public class Pattern extends DittedBitSequence {
}
public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, MatchAction[] matchArray) {
public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray,
MatchAction[] matchArray) {
super(seq);
markOffset = offset;
postrule = postArray;

4
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternFactory.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +15,9 @@
*/
package ghidra.util.bytesearch;
/**
* Interface for factories that create Match Pattern classes
*/
public interface PatternFactory {
public MatchAction getMatchActionByName(String nm);

41
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternPairSet.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java
View file

@ -23,10 +23,35 @@ import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* Two collections of patterns that are paired together to create larger patterns
* The final large patterns all must first match a pattern from the "pre" pattern collection
* followed immediately by a pattern from the "post" pattern collection
* A set of "pre" DittedBitSequences and a set of "post" Patterns are paired to form a larger pattern.
* To match, a sequence from the "pre" sequence set must first match, then one of the "post" patterns
* is matched relative to the matching "pre" pattern. This class is really a storage object for the
* patterns and provides a mechanism to read the pre/post patterns from an XML file.
*
* The larger pattern has the idea of bits of check, which means the number of bits that are fixed to
* a value when matching (not don't care). There is a pre pattern bits of check and post pattern bits
* of check. The bits of check are used to statistically gauge the accuracy of the pattern.
*
* An example of the XML format follows:
* <patternpairs totalbits="32" postbits="16">
* <prepatterns>
* <data>0xe12fff1. </data>
* <data>0xe12fff1e 0x46c0 </data>
* <data>0xe12fff1e 0xe1a00000 </data>
* </prepatterns>
*
* <postpatterns>
* <data> 0xe24dd... 11101001 00101101 .1...... ....0000 </data>
* <data> 11101001 00101101 .1...... ....0000 0xe24dd... </data>
* <data> 11101001 00101101 .1...... ....0000 0x........ 0xe24dd... </data>
* <align mark="0" bits="3"/>
* <setcontext name="TMode" value="0"/>
* <funcstart/>
* </postpatterns>
* </patternpairs>
*
* Note: The post Patterns can also have a set of rules that must be satisfied along with one of the
* Pattern DittedBitSequence matches.
*/
public class PatternPairSet {
private int totalBitsOfCheck; // Minimum number of bits of check in final patterns
@ -46,14 +71,14 @@ public class PatternPairSet {
if (postcheck < postBitsOfCheck) {
continue;
}
for(int j=0;j<preSequences.size();++j) {
DittedBitSequence prepattern = preSequences.get(j);
for (DittedBitSequence prepattern : preSequences) {
int precheck = prepattern.getNumFixedBits();
if (precheck + postcheck < totalBitsOfCheck) {
continue;
}
DittedBitSequence concat = prepattern.concatenate(postpattern);
Pattern finalpattern = new Pattern(concat,prepattern.getSize(),postpattern.getPostRules(),postpattern.getMatchActions());
Pattern finalpattern = new Pattern(concat, prepattern.getSize(),
postpattern.getPostRules(), postpattern.getMatchActions());
finalpats.add(finalpattern);
}
}
@ -99,8 +124,8 @@ public class PatternPairSet {
postRuleArray.toArray(postRules);
MatchAction[] matchActions = new MatchAction[matchActionArray.size()];
matchActionArray.toArray(matchActions);
for(int i=0;i<postdit.size();++i) {
Pattern postpat = new Pattern(postdit.get(i),0,postRules,matchActions);
for (DittedBitSequence element : postdit) {
Pattern postpat = new Pattern(element, 0, postRules, matchActions);
postPatterns.add(postpat);
}
parser.end(); // End postpatterns

4
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PostRule.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,6 +17,9 @@ package ghidra.util.bytesearch;
import ghidra.xml.XmlPullParser;
/**
* Inteface for post match rules that are checked after a match is idenfied
*/
public interface PostRule {
public boolean apply(Pattern pat, long matchoffset);

270
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java
View file

@ -15,17 +15,19 @@
*/
package ghidra.util.bytesearch;
import ghidra.util.task.TaskMonitor;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.*;
import ghidra.util.task.TaskMonitor;
/**
* SeqenceSearchState holds the state of a search for a DittedBitSequence within a byte
* sequence.
*/
public class SequenceSearchState implements Comparable<SequenceSearchState> {
private static final int PATERN_ENDED = 10000000;
private SequenceSearchState parent;
private ArrayList<DittedBitSequence> possible; // Patterns that could still match in this state
private ArrayList<DittedBitSequence> success; // Patterns that have matched successfully if we reached this state
@ -40,19 +42,21 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
public int getMaxSequenceSize() {
int max = 0;
for(int i=0;i<possible.size();++i) {
int val = possible.get(i).getSize();
if (val > max)
for (DittedBitSequence element : possible) {
int val = element.getSize();
if (val > max) {
max = val;
}
}
return max;
}
public void addSequence(DittedBitSequence pat, int pos) {
possible.add(pat);
if (pos == pat.getSize()) {
if (success == null)
if (success == null) {
success = new ArrayList<DittedBitSequence>();
}
success.add(pat);
}
}
@ -65,36 +69,40 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
};
Collections.sort(possible, comp);
if (success != null)
if (success != null) {
Collections.sort(success, comp);
}
}
@Override
public int compareTo(SequenceSearchState o) {
int i = 0;
for (;;) {
if (possible.size() <= i) {
if (o.possible.size() <=i)
if (o.possible.size() <= i) {
return 0;
}
return -1;
}
if (o.possible.size() <= i)
if (o.possible.size() <= i) {
return 1;
}
int indus = possible.get(i).getIndex(); // Lexicographic compare an sequence of sequences
int indthem = o.possible.get(i).getIndex();
if (indus != indthem)
if (indus != indthem) {
return (indus < indthem) ? -1 : 1;
}
i += 1;
}
}
private void buildSingleTransition(ArrayList<SequenceSearchState> all, int pos, int val) {
SequenceSearchState newstate = null;
for(int i=0;i<possible.size();++i) {
DittedBitSequence curpat = possible.get(i);
for (DittedBitSequence curpat : possible) {
if (curpat.isMatch(pos, val)) {
if (newstate == null)
if (newstate == null) {
newstate = new SequenceSearchState(this);
}
newstate.addSequence(curpat, pos + 1);
}
}
@ -106,8 +114,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
private void exportSuccess(ArrayList<Match> match, int offset) {
for(int i=0;i<success.size();++i) { // If we found matches
Match newmatch = new Match(success.get(i),offset);
for (DittedBitSequence succes : success) { // If we found matches
Match newmatch = new Match(succes, offset);
match.add(newmatch);
}
}
@ -119,9 +127,10 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
private void merge(SequenceSearchState op) {
SequenceSearchState parent = op.parent;
for (int i = 0; i < 256; ++i) {
if (parent.trans[i] == op) // Any references to -op- in parent
if (parent.trans[i] == op) {
parent.trans[i] = this; // Should be replaced with this
}
}
if (op.success != null) { // Merge
if (success == null) {
success = op.success;
@ -131,8 +140,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
int i = 0;
int j = 0;
int curpat = -1;
int thispat = success.get(i).index;
int oppat = op.success.get(j).index;
int thispat = success.get(i).getIndex();
int oppat = op.success.get(j).getIndex();
while ((i < success.size()) || (j < op.success.size())) {
if (thispat == oppat) {
if (curpat != thispat) {
@ -141,8 +150,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
}
i += 1;
j += 1;
thispat = (i==success.size()) ? 10000000 : success.get(i).index;
oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
oppat =
(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
}
else if (thispat < oppat) {
if (curpat != thispat) {
@ -150,7 +160,7 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
curpat = thispat;
}
i += 1;
thispat = (i==success.size()) ? 10000000 : success.get(i).index;
thispat = (i == success.size()) ? PATERN_ENDED : success.get(i).getIndex();
}
else {
if (curpat != oppat) {
@ -158,7 +168,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
curpat = oppat;
}
j += 1;
oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
oppat =
(j == op.success.size()) ? PATERN_ENDED : op.success.get(j).getIndex();
}
}
success = tmp;
@ -171,12 +182,16 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
SequenceSearchState curstate = this;
do {
if (curstate.success != null)
if (curstate.success != null) {
curstate.exportSuccess(match, 0);
if (subindex >= numbytes) return;
}
if (subindex >= numbytes) {
return;
}
curstate = curstate.trans[0xff & bytearray[subindex]]; // Perform state transition based on next byte in buffer
subindex += 1;
} while(curstate != null);
}
while (curstate != null);
}
@ -192,14 +207,16 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
curstate = this; // New starting offset -> Root state
subindex = offset;
do {
if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point
if (curstate.success != null) {
curstate.exportSuccess(match, offset);
}
if (subindex >= buffer.length) { // if we've run out of bytes, must restart at next offset
break;
}
curstate = curstate.trans[0xff & buffer[subindex]]; // Perform state transition based on next byte
subindex += 1;
} while(curstate != null);
}
while (curstate != null);
}
}
@ -210,7 +227,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in,ArrayList<Match> match,TaskMonitor monitor) throws IOException {
public void apply(InputStream in, ArrayList<Match> match, TaskMonitor monitor)
throws IOException {
apply(in, -1L, match, monitor);
}
@ -222,119 +240,141 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in, long maxBytes, ArrayList<Match> match,TaskMonitor monitor) throws IOException {
int maxsize = getMaxSequenceSize()+1;
if (maxsize <4096)
maxsize = 4096;
public void apply(InputStream in, long maxBytes, ArrayList<Match> match, TaskMonitor monitor)
throws IOException {
int maxSize = getMaxSequenceSize() + 1;
if (maxSize < 4096) {
maxSize = 4096;
}
if (maxBytes > 0) {
maxBytes += getMaxSequenceSize() + 1;
}
byte[] firstbuf=new byte[maxsize];
byte[] secondbuf=new byte[maxsize];
byte[] curbuf;
SequenceSearchState curstate;
int fullbuffers; // Number of buffers that are completely full
int ra = in.read(firstbuf);
if (ra == firstbuf.length) {
ra = in.read(secondbuf);
if (ra == secondbuf.length) {
fullbuffers = 2;
byte[] firstBuf = new byte[maxSize];
byte[] secondBuf = new byte[maxSize];
byte[] curBuf;
SequenceSearchState curState;
int fullBuffers; // Number of buffers that are completely full
int ra = in.read(firstBuf);
if (ra == firstBuf.length) {
ra = in.read(secondBuf);
if (ra == secondBuf.length) {
fullBuffers = 2;
}
else {
if (ra < 0)
if (ra < 0) {
ra = 0;
fullbuffers = 1;
}
fullBuffers = 1;
byte[] tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = secondbuf[i];
secondbuf = tmp;
for (int i = 0; i < ra; ++i) {
tmp[i] = secondBuf[i];
}
secondBuf = tmp;
}
}
else if (ra < 0)
else if (ra < 0) {
return; // No bytes at all were read
}
else {
byte[] tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = firstbuf[i];
firstbuf = tmp;
fullbuffers = 0;
secondbuf = new byte[0];
for (int i = 0; i < ra; ++i) {
tmp[i] = firstBuf[i];
}
firstBuf = tmp;
fullBuffers = 0;
secondBuf = new byte[0];
}
int offset = 0;
int bufreloff=0;
int subindex;
while(fullbuffers == 2) {
curstate = this; // New starting offset -> Root state
subindex = bufreloff;
curbuf = firstbuf;
int bufRelativeOffset = 0;
int subIndex;
while (fullBuffers == 2) {
curState = this; // New starting offset -> Root state
subIndex = bufRelativeOffset;
curBuf = firstBuf;
do {
if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point
curstate.exportSuccess(match, offset);
if (subindex >= curbuf.length) { // check that we have enough bytes in current buffer
curbuf = secondbuf; // If not, switch to secondary buffer
subindex = 0;
if (curState.success != null) {
curState.exportSuccess(match, offset);
}
curstate = curstate.trans[ 0xff & curbuf[subindex] ]; // Perform state transition based on next byte in buffer
subindex += 1;
} while(curstate != null);
if (subIndex >= curBuf.length) { // check that we have enough bytes in current buffer
curBuf = secondBuf; // If not, switch to secondary buffer
subIndex = 0;
}
curState = curState.trans[0xff & curBuf[subIndex]]; // Perform state transition based on next byte in buffer
subIndex += 1;
}
while (curState != null);
offset += 1; // Advance to next starting offset
if (maxBytes > 0 && offset > maxBytes) {
break;
}
bufreloff += 1;
if (bufreloff == firstbuf.length) { // If starting offset no longer falls in firstbuf
byte[] tmp = firstbuf; // Switch firstbuf with secondbuf
firstbuf = secondbuf;
secondbuf = tmp;
ra = in.read(secondbuf); // refill secondbuf (old firstbuf) with new bytes
bufRelativeOffset += 1;
if (bufRelativeOffset == firstBuf.length) { // If starting offset no longer falls in firstbuf
byte[] tmp = firstBuf; // Switch firstbuf with secondbuf
firstBuf = secondBuf;
secondBuf = tmp;
ra = in.read(secondBuf); // refill secondbuf (old firstbuf) with new bytes
if (monitor != null) {
if (monitor.isCancelled()) return;
if (monitor.isCancelled()) {
return;
}
monitor.setProgress(offset);
}
if (ra != secondbuf.length) {
fullbuffers = 1;
if (ra < 0)
if (ra != secondBuf.length) {
fullBuffers = 1;
if (ra < 0) {
ra = 0;
}
tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = secondbuf[i];
secondbuf = tmp;
for (int i = 0; i < ra; ++i) {
tmp[i] = secondBuf[i];
}
bufreloff = 0;
secondBuf = tmp;
}
bufRelativeOffset = 0;
}
}
while(fullbuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
if (secondbuf.length == 0)
fullbuffers = 0;
curstate = this;
subindex = bufreloff;
curbuf = firstbuf;
while (fullBuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
if (secondBuf.length == 0) {
fullBuffers = 0;
}
curState = this;
subIndex = bufRelativeOffset;
curBuf = firstBuf;
do {
if (curstate.success != null)
curstate.exportSuccess(match, offset);
if (subindex >= curbuf.length) {
if (curbuf == secondbuf) break; // Out of data, all pending patterns fail
curbuf = secondbuf;
subindex = 0;
if (curbuf.length==0) break;
if (curState.success != null) {
curState.exportSuccess(match, offset);
}
curstate = curstate.trans[ 0xff & curbuf[subindex] ];
subindex += 1;
} while(curstate != null);
if (subIndex >= curBuf.length) {
if (curBuf == secondBuf) {
break; // Out of data, all pending patterns fail
}
curBuf = secondBuf;
subIndex = 0;
if (curBuf.length == 0) {
break;
}
}
curState = curState.trans[0xff & curBuf[subIndex]];
subIndex += 1;
}
while (curState != null);
offset += 1;
bufreloff += 1;
if (bufreloff == firstbuf.length) {
if (fullbuffers == 0) break;
firstbuf = secondbuf;
fullbuffers = 0;
bufreloff = 0;
secondbuf = new byte[0];
bufRelativeOffset += 1;
if (bufRelativeOffset == firstBuf.length) {
if (fullBuffers == 0) {
break;
}
firstBuf = secondBuf;
fullBuffers = 0;
bufRelativeOffset = 0;
secondBuf = new byte[0];
}
}
}
static public ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,int pos) {
static public ArrayList<SequenceSearchState> buildTransitionLevel(
ArrayList<SequenceSearchState> prev, int pos) {
ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iterator = prev.iterator();
while (iterator.hasNext()) { // For each current state
@ -344,7 +384,9 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
next.buildSingleTransition(res, pos, i);
}
}
if (res.isEmpty()) return res;
if (res.isEmpty()) {
return res;
}
// Prepare to dedup the states
Collections.sort(res);
ArrayList<SequenceSearchState> finalres = new ArrayList<SequenceSearchState>();
@ -365,11 +407,12 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
return finalres;
}
static public SequenceSearchState buildStateMachine(ArrayList<? extends DittedBitSequence> patterns) {
static public SequenceSearchState buildStateMachine(
ArrayList<? extends DittedBitSequence> patterns) {
SequenceSearchState root = new SequenceSearchState(null);
for (int i = 0; i < patterns.size(); ++i) {
DittedBitSequence pat = patterns.get(i);
pat.index = i;
pat.setIndex(i);
root.addSequence(pat, 0);
}
root.sortSequences();
@ -379,7 +422,8 @@ public class SequenceSearchState implements Comparable<SequenceSearchState> {
do {
statelevel = buildTransitionLevel(statelevel, level);
level += 1;
} while(!statelevel.isEmpty());
}
while (!statelevel.isEmpty());
return root;
}
}

26
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java
View file

@ -141,7 +141,8 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
* @return true if the data type creation completes successfully.
* @throws CancelledException if the user cancels this task.
*/
private boolean doApplyTo(Program program, TaskMonitor taskMonitor) throws CancelledException {
protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
throws CancelledException {
try {
monitor = taskMonitor;
@ -177,7 +178,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
catch (InvalidInputException e) {
// Catch the exception and output the error, but still should create
// associated data if possible, even though markup failed.
handleErrorMessage(program, name, address, address, e);
handleErrorMessage(program, name, getDataAddress(), getDataAddress(), e);
success = false;
}
@ -190,7 +191,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
}
catch (AddressOutOfBoundsException | CodeUnitInsertionException | DataTypeConflictException
| InvalidDataTypeException e) {
handleErrorMessage(program, name, address, address, e);
handleErrorMessage(program, name, getDataAddress(), getDataAddress(), e);
return false;
}
}
@ -211,8 +212,8 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
throw new CodeUnitInsertionException(
"Unable to get data type from model, " + model.getName() + ".");
}
if (!memory.getLoadedAndInitializedAddressSet().contains(address)) {
String message = "Can't create an " + dt.getName() + " @ " + address +
if (!memory.getLoadedAndInitializedAddressSet().contains(getDataAddress())) {
String message = "Can't create an " + dt.getName() + " @ " + getDataAddress() +
" which isn't in loaded and initialized memory for " + program.getName();
throw new CodeUnitInsertionException(message);
}
@ -228,14 +229,15 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
monitor.checkCanceled();
// Is the data type already applied at the address?
if (matchingDataExists(dt, program, address)) {
if (matchingDataExists(dt, program, getDataAddress())) {
return;
}
monitor.checkCanceled();
// Create data at the address using the datatype.
DataUtilities.createData(program, address, dt, dt.getLength(), false, getClearDataMode());
DataUtilities.createData(program, getDataAddress(), dt, dt.getLength(), false,
getClearDataMode());
}
/**
@ -353,4 +355,14 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
return (applyOptions.shouldClearDefinedData()) ? ClearDataMode.CLEAR_ALL_CONFLICT_DATA
: ClearDataMode.CLEAR_ALL_UNDEFINED_CONFLICT_DATA;
}
/**
* Get the address for the data item to be processed by the base implementation.
* In general this is the initial model address set when the command was created.
*
* @return the address of the data item being created.
*/
protected Address getDataAddress() {
return address;
}
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/CreateTypeDescriptorBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateTypeDescriptorBackgroundCmd
private void loadModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new TypeDescriptorModel(program, address, validationOptions);
model = new TypeDescriptorModel(program, getDataAddress(), validationOptions);
}
}
@ -138,13 +138,13 @@ public class CreateTypeDescriptorBackgroundCmd
String prefix = demangledName + " ";
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, address,
EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
applyOptions);
monitor.checkCanceled();
// Label
EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, address,
EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
applyOptions);
return true;

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHCatchHandlerMapBackgroundCmd.java
View file

@ -74,7 +74,7 @@ public class CreateEHCatchHandlerMapBackgroundCmd
@Override
protected EHCatchHandlerModel createModel(Program program) {
if (model == null) {
model = new EHCatchHandlerModel(program, count, address, validationOptions);
model = new EHCatchHandlerModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHESTypeListBackgroundCmd.java
View file

@ -70,7 +70,7 @@ public class CreateEHESTypeListBackgroundCmd
@Override
protected EHESTypeListModel createModel(Program program) {
if (model == null) {
model = new EHESTypeListModel(program, address, validationOptions);
model = new EHESTypeListModel(program, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHFuncInfoBackgroundCmd.java
View file

@ -71,7 +71,7 @@ public class CreateEHFuncInfoBackgroundCmd
@Override
protected EHFunctionInfoModel createModel(Program program) {
if (model == null) {
model = new EHFunctionInfoModel(program, address, validationOptions);
model = new EHFunctionInfoModel(program, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHIPToStateMapBackgroundCmd.java
View file

@ -72,7 +72,7 @@ public class CreateEHIPToStateMapBackgroundCmd
@Override
protected EHIPToStateModel createModel(Program program) {
if (model == null) {
model = new EHIPToStateModel(program, count, address, validationOptions);
model = new EHIPToStateModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHTryBlockMapBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateEHTryBlockMapBackgroundCmd
@Override
protected EHTryBlockModel createModel(Program program) {
if (model == null) {
model = new EHTryBlockModel(program, count, address, validationOptions);
model = new EHTryBlockModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHUnwindMapBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateEHUnwindMapBackgroundCmd extends AbstractCreateDataBackground
@Override
protected EHUnwindModel createModel(Program program) {
if (model == null) {
model = new EHUnwindModel(program, count, address, validationOptions);
model = new EHUnwindModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

8
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti1BackgroundCmd.java
View file

@ -63,7 +63,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti1Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti1Model(program, address, validationOptions);
model = new Rtti1Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -101,14 +101,14 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
}
catch (InvalidDataTypeException e) {
// Couldn't get pmd and attributes so leave it off and simply log the error.
String message = "Unable to get PMD and attributes for RTTI1 at " + address + ".";
String message = "Unable to get PMD and attributes for RTTI1 at " + getDataAddress() + ".";
handleError(message);
}
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_1_NAME, suffix, address, applyOptions);
RTTI_1_NAME, suffix, getDataAddress(), applyOptions);
monitor.checkCanceled();
@ -116,7 +116,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
if (applyOptions.shouldCreateLabel()) {
String rtti1Suffix = RTTI_1_NAME + suffix;
rtti1Suffix = SymbolUtilities.replaceInvalidChars(rtti1Suffix, true);
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, rtti1Suffix);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, rtti1Suffix);
}
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti2BackgroundCmd.java
View file

@ -68,7 +68,7 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti2Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti2Model(program, rtti1Count, address, validationOptions);
model = new Rtti2Model(program, rtti1Count, getDataAddress(), validationOptions);
}
return model;
}
@ -118,13 +118,13 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_2_NAME, null, address, applyOptions);
RTTI_2_NAME, null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_2_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_2_NAME);
}
return true;
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti3BackgroundCmd.java
View file

@ -63,7 +63,7 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti3Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti3Model(program, address, validationOptions);
model = new Rtti3Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -105,13 +105,13 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_3_NAME, null, address, applyOptions);
RTTI_3_NAME, null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_3_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_3_NAME);
}
}

203
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti4BackgroundCmd.java
View file

@ -15,29 +15,34 @@
*/
package ghidra.app.cmd.data.rtti;
import java.util.List;
import java.util.Set;
import java.util.*;
import ghidra.app.cmd.data.*;
import ghidra.app.util.datatype.microsoft.DataApplyOptions;
import ghidra.app.util.datatype.microsoft.DataValidationOptions;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.InvalidDataTypeException;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.model.symbol.Namespace;
import ghidra.program.util.ProgramMemoryUtil;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* This command will create an RTTI4 data type.
* If there are any existing instructions in the area to be made into data, the command will fail.
* This command will create multiple RTTI4 data types all at one time.
* If there are any existing instructions in the areas to be made into data, the command will fail only on that RTTI4 entry.
* Any data in the area will be replaced with the new dataType.
*/
public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rtti4Model> {
private static final String RTTI_4_NAME = "RTTI Complete Object Locator";
private List<MemoryBlock> vfTableBlocks;
private List<Address> rtti4Locations;
private Address currentProcessingAddress;
/**
* Constructs a command for applying an RTTI4 dataType at an address.
@ -54,12 +59,44 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
super(Rtti4Model.DATA_TYPE_NAME, address, 1, validationOptions, applyOptions);
this.vfTableBlocks = vfTableBlocks;
rtti4Locations = new ArrayList<Address>();
rtti4Locations.add(address);
}
public CreateRtti4BackgroundCmd(List<Address> addresses, List<MemoryBlock> vfTableBlocks,
DataValidationOptions validationOptions, DataApplyOptions applyOptions) {
super(Rtti4Model.DATA_TYPE_NAME, addresses.get(0), 1, validationOptions, applyOptions);
this.rtti4Locations = addresses;
this.vfTableBlocks = vfTableBlocks;
}
@Override
protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
throws CancelledException {
// process each potential RTTI4 entry, keeping track of good RTTI4 tables
List<Address> goodRtti4Locations = new ArrayList<Address>();
boolean succeeded = false;
for (Address addr : rtti4Locations) {
currentProcessingAddress = addr;
succeeded |= super.doApplyTo(program, taskMonitor);
goodRtti4Locations.add(addr);
}
// if any succeeded and should create associated data, make the vftables all at one time
if (succeeded && applyOptions.shouldFollowData()) {
createaAssociatedVfTables(program, goodRtti4Locations, taskMonitor);
}
return succeeded;
}
@Override
protected Rtti4Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti4Model(program, address, validationOptions);
if (model == null || program != model.getProgram() ||
!getDataAddress().equals(model.getAddress())) {
model = new Rtti4Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -75,7 +112,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
}
catch (InvalidDataTypeException e) {
createRtti0Success = false;
// log message and continue with other markup.
// log message and continue with other mark-up.
handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
}
@ -89,9 +126,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
}
boolean createVfTableSuccess = createVfTable();
return createRtti0Success && createRtti3Success && createVfTableSuccess;
return createRtti0Success && createRtti3Success;
}
private boolean createRtti0() throws CancelledException, InvalidDataTypeException {
@ -112,57 +147,99 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
return cmd.applyTo(model.getProgram(), monitor);
}
private boolean createVfTable() throws CancelledException {
private boolean createaAssociatedVfTables(Program program, List<Address> goodRtti4Locations,
TaskMonitor taskMonitor) throws CancelledException {
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables");
HashMap<Address, VfTableModel> foundVFtables = new HashMap<>();
for (Address rtti4Address : goodRtti4Locations) {
byte[] bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti4Address);
addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti4Address);
addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
}
AddressSet searchSet = new AddressSet();
for (MemoryBlock block : vfTableBlocks) {
searchSet.add(block.getStart(), block.getEnd());
}
searcher.search(program, searchSet, monitor);
// did the search, now process the results
boolean didSome = false;
for (Address rtti4Address : goodRtti4Locations) {
monitor.checkCanceled();
Program program = model.getProgram();
Address rtti4Address = address;
int defaultPointerSize = program.getDefaultPointerSize();
int alignment = defaultPointerSize; // Align the vf table based on the size of the pointers in it.
Set<Address> directRtti4Refs =
ProgramMemoryUtil.findDirectReferences(program, vfTableBlocks, alignment, rtti4Address,
monitor);
VfTableModel validVfTableModel = null;
for (Address possibleVfMetaAddr : directRtti4Refs) {
monitor.checkCanceled();
Address possibleVfTableAddr = possibleVfMetaAddr.add(defaultPointerSize);
// Validate the model. Don't apply the command if invalid.
try {
VfTableModel vfTableModel =
new VfTableModel(program, possibleVfTableAddr, validationOptions);
vfTableModel.validate();
if (validVfTableModel != null) {
String message = "More than one possible vfTable found for " +
Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(program, rtti4Address, message);
return false;
}
validVfTableModel = vfTableModel;
}
catch (InvalidDataTypeException e) {
continue; // This isn't a valid model.
}
}
if (validVfTableModel == null) {
VfTableModel vfTableModel = foundVFtables.get(rtti4Address);
if (vfTableModel == null) {
String message =
"No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(program, rtti4Address, message);
return false;
continue;
}
monitor.checkCanceled();
CreateVfTableBackgroundCmd cmd =
new CreateVfTableBackgroundCmd(validVfTableModel, applyOptions);
return cmd.applyTo(program, monitor);
new CreateVfTableBackgroundCmd(vfTableModel, applyOptions);
didSome |= cmd.applyTo(program, monitor);
}
return didSome;
}
/**
* Add a search pattern, to the searcher, for the set of bytes representing an rtti4 location.
* Only one VFTable for is allowed for an RTT4 location, last one in wins and gets created.
*
* @param searcher byte pattern searcher
* @param foundVFtables list of addresses accumulated when actual search is performed
* @param rtti4Address location of rttiAddress to find vfTable for
* @param bytes bytes representing rtti4Addres to be found in memory
*/
private void addByteSearchPattern(MemoryBytePatternSearcher searcher,
HashMap<Address, VfTableModel> foundVFtables, Address rtti4Address, byte[] bytes) {
if (bytes == null) {
return;
}
GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti4Address) {
@Override
public void apply(Program prog, Address addr, Match match) {
Address possibleVfTableAddr = addr.add(prog.getDefaultPointerSize());
// See if VfTable is valid, and add to rtti4 to vfTable map
try {
VfTableModel vfTableModel =
new VfTableModel(prog, possibleVfTableAddr, validationOptions);
vfTableModel.validate();
VfTableModel existing = foundVFtables.put(rtti4Address, vfTableModel);
if (existing != null) {
// potential table already found, is an error, don't know which is right
String message = "More than one possible vfTable found for " +
Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(prog, rtti4Address, message);
}
}
catch (InvalidDataTypeException e) {
// This isn't a valid model.
}
}
};
GenericByteSequencePattern<Address> genericByteMatchPattern =
new GenericByteSequencePattern<Address>(bytes, action);
searcher.addPattern(genericByteMatchPattern);
}
@Override
@ -178,19 +255,31 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
if (rtti0Model != null) {
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_4_NAME, null, address, applyOptions);
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.CONST_PREFIX + RttiUtil.getDescriptorTypeNamespace(rtti0Model) +
Namespace.DELIMITER,
RTTI_4_NAME, null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_4_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model,
RTTI_4_NAME);
}
}
return true;
}
/**
* Creating RTTI4 tables handles creating multiple tables at once.
* The generic base class that handles getting the address will get the RTTI4 tabl address.
*
* @return the current address to process
*/
@Override
protected Address getDataAddress() {
return currentProcessingAddress;
}
}

10
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateVfTableBackgroundCmd.java
View file

@ -70,7 +70,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
@Override
protected VfTableModel createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new VfTableModel(program, address, validationOptions);
model = new VfTableModel(program, getDataAddress(), validationOptions);
}
return model;
}
@ -105,7 +105,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
return false;
}
long displacement = dataType.getLength();
Address terminatorAddress = address.add(displacement);
Address terminatorAddress = getDataAddress().add(displacement);
try {
Address referencedAddress = getAbsoluteAddress(program, terminatorAddress);
if (referencedAddress == null || referencedAddress.getOffset() != 0) {
@ -136,7 +136,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
private boolean createMetaPointer() {
Program program = model.getProgram();
Address metaAddress = address.subtract(program.getDefaultPointerSize());
Address metaAddress = getDataAddress().subtract(program.getDefaultPointerSize());
// Create a pointer to the RTTI4 associated with the vf table.
DataType metaPointer = new PointerDataType(program.getDataTypeManager());
@ -162,7 +162,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
private boolean createVfTableMarkup() throws CancelledException, InvalidDataTypeException {
Address vfTableAddress = address;
Address vfTableAddress = getDataAddress();
Program program = model.getProgram();
monitor.checkCanceled();
@ -238,7 +238,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
* the vftable.
*/
private Address getMetaAddress(Program program) {
return address.subtract(program.getDefaultPointerSize());
return getDataAddress().subtract(program.getDefaultPointerSize());
}
}

8
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/VfTableModel.java
View file

@ -15,7 +15,7 @@
*/
package ghidra.app.cmd.data.rtti;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.getAbsoluteAddress;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.*;
import ghidra.app.cmd.data.AbstractCreateDataTypeModel;
import ghidra.app.cmd.data.TypeDescriptorModel;
@ -57,6 +57,8 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
DataValidationOptions validationOptions) {
super(program, RttiUtil.getVfTableCount(program, vfTableAddress), vfTableAddress,
validationOptions);
// super's count will hold the number of valid address elements from getVfTableCount()
elementCount = getCount();
}
@Override
@ -80,7 +82,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
long entrySize = individualEntryDataType.getLength();
// Each entry is a pointer to where a function can possibly be created.
long numEntries = RttiUtil.getVfTableCount(program, startAddress);
long numEntries = getElementCount();
if (numEntries == 0) {
throw new InvalidDataTypeException(
getName() + " data type at " + getAddress() + " doesn't have a valid vf table.");
@ -123,7 +125,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
lastDataType = null;
lastElementCount = -1;
lastElementCount = RttiUtil.getVfTableCount(program, getAddress());
lastElementCount = getElementCount();
if (lastElementCount > 0) {
DataTypeManager dataTypeManager = program.getDataTypeManager();
PointerDataType pointerDt = new PointerDataType(dataTypeManager);

155
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java
View file

@ -30,6 +30,7 @@ import ghidra.program.model.lang.UndefinedValueException;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.util.ProgramMemoryUtil;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -41,7 +42,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
private static final String NAME = "Windows x86 PE RTTI Analyzer";
private static final String DESCRIPTION =
"This analyzer finds and creates all of the RTTI metadata structures and their associated vf tables.";
"Finds and creates RTTI metadata structures and associated vf tables.";
// TODO If we want the RTTI analyzer to find all type descriptors regardless of whether
// they are used for RTTI, then change the CLASS_PREFIX_CHARS to ".". Need to be
@ -133,6 +134,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
monitor.setMaximum(possibleRtti0Addresses.size());
monitor.setMessage("Creating RTTI Data...");
ArrayList<Address> rtti0Locations = new ArrayList<Address>();
int count = 0;
for (Address rtti0Address : possibleRtti0Addresses) {
monitor.checkCanceled();
@ -157,97 +159,164 @@ public class RttiAnalyzer extends AbstractAnalyzer {
rtti0Address, validationOptions, applyOptions);
typeDescCmd.applyTo(program, monitor);
rtti0Locations.add(rtti0Address);
}
// Create any valid RTTI4s for this TypeDescriptor
processRtti4sForRtti0(program, rtti0Address, monitor);
}
processRtti4sForRtti0(program, rtti0Locations, monitor);
}
private void processRtti4sForRtti0(Program program, Address rtti0Address, TaskMonitor monitor)
throws CancelledException {
private void processRtti4sForRtti0(Program program, List<Address> rtti0Locations,
TaskMonitor monitor) throws CancelledException {
List<MemoryBlock> rDataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
List<MemoryBlock> dataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
program.getMemory(), ".rdata", monitor);
dataBlocks.addAll(ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
program.getMemory(), ".data", monitor));
List<Address> rtti4Addresses =
getRtti4Addresses(program, rDataBlocks, rtti0Address, validationOptions, monitor);
getRtti4Addresses(program, dataBlocks, rtti0Locations, validationOptions, monitor);
for (Address rtti4Address : rtti4Addresses) {
monitor.checkCanceled();
CreateRtti4BackgroundCmd cmd =
new CreateRtti4BackgroundCmd(rtti4Address, rDataBlocks, validationOptions,
applyOptions);
// create all found RTTI4 tables at once
CreateRtti4BackgroundCmd cmd = new CreateRtti4BackgroundCmd(rtti4Addresses, dataBlocks,
validationOptions, applyOptions);
cmd.applyTo(program, monitor);
}
}
/**
* Gets the base addresses of all the RTTI 4 structures that appear to be associated with
* the RTTI 0 at the indicated base address.
* @param program the program containing the RTTI 0 data structure.
* @param rtti4Blocks the memory blocks to be searched for RTTI4 structures.
* @param rtti0Address the base address of the RTTI 0 structure in the program
* @param rtti0Locations the base addresses of the RTTI 0 structure in the program
* @param validationOptions options indicating how validation is performed for data structures
* @param monitor the task monitor for cancelling a task
* @param monitor the task monitor for canceling a task
* @return the RTTI 4 base addresses associated with the RTTI 0
* @throws CancelledException if the user cancels this task.
*/
private static List<Address> getRtti4Addresses(Program program, List<MemoryBlock> rtti4Blocks,
Address rtti0Address, DataValidationOptions validationOptions, TaskMonitor monitor)
throws CancelledException {
List<Address> rtti0Locations, DataValidationOptions validationOptions,
TaskMonitor monitor) throws CancelledException {
monitor.checkCanceled();
List<Address> addresses =
getRefsToRtti0(program, rtti4Blocks, rtti0Locations, validationOptions, monitor);
return addresses;
}
/** For each of the RTTI0 locations found locate the associated RTTI4 structure referring to it.
*
* @param program program to be searched
* @param dataBlocks dataBlocks to search
* @param rtti0Locations list of known rtti0 locations
* @param validationOptions options for validation of found RTTI4 entries
* @param monitor to cancel
* @return list of found RTTI4 references to known RTTI0 locations
* @throws CancelledException if canceled
*/
private static List<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
List<Address> rtti0Locations, DataValidationOptions validationOptions,
TaskMonitor monitor) throws CancelledException {
List<Address> addresses = new ArrayList<>(); // the RTTI 4 addresses
int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset();
Set<Address> refsToRtti0 = getRefsToRtti0(program, rtti4Blocks, rtti0Address);
// for each RTTI 0 now see if we can get RTTI4s that refer to it.
for (Address refAddress : refsToRtti0) {
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI0 refernces");
monitor.checkCanceled();
for (Address rtti0Address : rtti0Locations) {
byte[] bytes;
if (MSDataTypeUtils.is64Bit(program)) {
// 64-bit programs will have the addresses as offsets from the image base (BOS)
bytes = ProgramMemoryUtil.getImageBaseOffsets32Bytes(program, 4, rtti0Address);
addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
rtti0Address, bytes);
}
else {
// 32-bit could have direct address in memory
bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti0Address);
addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
rtti0Address, bytes);
// or referenced by a shifted pointer based on program data organization shift amount
bytes = ProgramMemoryUtil.getShiftedDirectAddressBytes(program, rtti0Address);
addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
rtti0Address, bytes);
}
}
AddressSet searchSet = new AddressSet();
for (MemoryBlock block : dataBlocks) {
searchSet.add(block.getStart(), block.getEnd());
}
searcher.search(program, searchSet, monitor);
return addresses;
}
/**
* Add a search pattern, to the searcher, for the set of bytes representing an address
*
* @param searcher pattern searcher
* @param validationOptions RTTI4 validation options
* @param addresses list of found valid RTTI4 locations accumulated during actual search
* @param rtti0PointerOffset offset of pointer in RTTI4 entry to RTTI0
* @param rtti0Address RTTI0 address corresponding to pattern of bytes
* @param bytes pattern of bytes in memory corresponding to address
*/
private static void addByteSearchPattern(MemoryBytePatternSearcher searcher,
DataValidationOptions validationOptions, List<Address> addresses,
int rtti0PointerOffset, Address rtti0Address, byte[] bytes) {
// no pattern bytes.
if (bytes == null) {
return;
}
// Each time a match for this byte pattern validate as an RTTI4 and add to list
GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti0Address) {
@Override
public void apply(Program prog, Address addr, Match match) {
Address possibleRtti4Address;
try {
possibleRtti4Address = refAddress.subtractNoWrap(rtti0PointerOffset);
possibleRtti4Address = addr.subtractNoWrap(rtti0PointerOffset);
}
catch (AddressOverflowException e) {
continue; // Couldn't get an Rtti4 address.
return; // Couldn't get an Rtti4 address.
}
Rtti4Model rtti4Model =
new Rtti4Model(program, possibleRtti4Address, validationOptions);
new Rtti4Model(prog, possibleRtti4Address, validationOptions);
try {
rtti4Model.validate();
}
catch (InvalidDataTypeException e) {
continue; // Only process valid RTTI 4 data.
return; // Only process valid RTTI 4 data.
}
// Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly
// through the RTTI 3.
boolean refersToRtti0 = rtti4Model.refersToRtti0(rtti0Address);
boolean refersToRtti0 = rtti4Model.refersToRtti0(getMatchValue());
if (!refersToRtti0) {
continue; // Only process valid RTTI 4 data.
return; // Only process valid RTTI 4 data.
}
// add to list of RTTI4 locations to be processed later
addresses.add(possibleRtti4Address);
}
return addresses;
}
};
private static Set<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
Address rtti0Address) throws CancelledException {
Set<Address> refsToRtti0;
if (MSDataTypeUtils.is64Bit(program)) {
refsToRtti0 = ProgramMemoryUtil.findImageBaseOffsets32(program, 4, rtti0Address,
TaskMonitor.DUMMY);
}
else {
refsToRtti0 = ProgramMemoryUtil.findDirectReferences(program, dataBlocks,
program.getDefaultPointerSize(), rtti0Address, TaskMonitor.DUMMY);
}
return refsToRtti0;
// create a Pattern of the bytes and the MatchAction to perform upon a match
GenericByteSequencePattern<Address> genericByteMatchPattern =
new GenericByteSequencePattern<Address>(bytes, action);
searcher.addPattern(genericByteMatchPattern);
}
}