yetangitu/ghidra
1
0
Fork 0
mirror of https://github.com/NationalSecurityAgency/ghidra.git synced 2025-10-04 10:19:23 +02:00
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/GT-3341_emteere_RTTIPerformance'

Browse source
This commit is contained in:
ghidorahrex 2020-02-19 12:06:25 -05:00
commit 024a6190e0
35 changed files with 1692 additions and 907 deletions
Download patch file Download diff file Expand all files Collapse all files

142
Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/analysis/EmbeddedMediaAnalyzer.java
View file

@ -15,23 +15,27 @@
*/
package ghidra.app.plugin.core.analysis;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.cmd.data.CreateDataCmd;
import ghidra.app.services.*;
import ghidra.app.util.importer.MessageLog;
import ghidra.framework.options.Options;
import ghidra.program.model.address.*;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSetView;
import ghidra.program.model.data.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.listing.BookmarkType;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.Memory;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
private static final String NAME = "Embedded Media";
private static final String DESCRIPTION =
"Finds and tries to apply embedded media data types (ie png, gif, jpeg, wav) in current program.";
"Finds embedded media data types (ie png, gif, jpeg, wav)";
private static final String OPTION_NAME_CREATE_BOOKMARKS = "Create Analysis Bookmarks";
private static final String OPTION_DESCRIPTION_CREATE_BOOKMARKS =
@ -52,92 +56,78 @@ public class EmbeddedMediaAnalyzer extends AbstractAnalyzer {
throws CancelledException {
Memory memory = program.getMemory();
AddressSetView initializedAddressSet = memory.getLoadedAndInitializedAddressSet();
AddressSet initialedSearchSet = set.intersect(initializedAddressSet);
AddressSetView validMemorySet = memory.getLoadedAndInitializedAddressSet();
AddressSetView searchSet = set.intersect(validMemorySet);
if (searchSet.isEmpty()) {
return false; // no valid addresses to search
}
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("Embedded Media");
List<Address> foundMedia = new ArrayList<>();
foundMedia = scanForMedia(program, new GifDataType(), "GIF 87", GifDataType.MAGIC_87,
GifDataType.GIFMASK, initialedSearchSet, memory, monitor);
addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 87",
GifDataType.MAGIC_87, GifDataType.GIFMASK);
foundMedia.addAll(scanForMedia(program, new GifDataType(), "GIF 89", GifDataType.MAGIC_89,
GifDataType.GIFMASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new GifDataType(), "GIF 89",
GifDataType.MAGIC_89, GifDataType.GIFMASK);
foundMedia.addAll(scanForMedia(program, new PngDataType(), "PNG", PngDataType.MAGIC,
PngDataType.MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new PngDataType(), "PNG",
PngDataType.MAGIC, PngDataType.MASK);
foundMedia.addAll(scanForMedia(program, new JPEGDataType(), "JPEG", JPEGDataType.MAGIC,
JPEGDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new JPEGDataType(), "JPEG",
JPEGDataType.MAGIC, JPEGDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new WAVEDataType(), "WAVE", WAVEDataType.MAGIC,
WAVEDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new WAVEDataType(), "WAVE",
WAVEDataType.MAGIC, WAVEDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new AUDataType(), "AU", AUDataType.MAGIC,
AUDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new AUDataType(), "AU",
AUDataType.MAGIC, AUDataType.MAGIC_MASK);
foundMedia.addAll(scanForMedia(program, new AIFFDataType(), "AIFF", AIFFDataType.MAGIC,
AIFFDataType.MAGIC_MASK, initialedSearchSet, memory, monitor));
addByteSearchPattern(searcher, program, foundMedia, new AIFFDataType(), "AIFF",
AIFFDataType.MAGIC, AIFFDataType.MAGIC_MASK);
return true;
searcher.search(program, searchSet, monitor);
return foundMedia.size() > 0;
}
private List<Address> scanForMedia(Program program, DataType dt, String mediaName,
byte[] mediaBytes, byte[] mask, AddressSetView addresses, Memory memory,
TaskMonitor monitor) {
monitor.setMessage("Scanning for " + mediaName + " Embedded Media");
monitor.initialize(addresses.getNumAddresses());
List<Address> foundMediaAddresses = new ArrayList<>();
Iterator<AddressRange> iterator = addresses.iterator();
while (iterator.hasNext()) {
if (monitor.isCancelled()) {
return foundMediaAddresses;
}
AddressRange range = iterator.next();
Address start = range.getMinAddress();
Address end = range.getMaxAddress();
Address found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
while (found != null && !monitor.isCancelled()) {
//See if it is already an applied media data type
Data data = program.getListing().getDefinedDataAt(found);
int skipLen = 1;
if (data == null) {
try {
CreateDataCmd cmd = new CreateDataCmd(found, dt);
if (cmd.applyTo(program)) {
if (createBookmarksEnabled) {
program.getBookmarkManager().setBookmark(found,
BookmarkType.ANALYSIS, "Embedded Media",
"Found " + mediaName + " Embedded Media");
}
foundMediaAddresses.add(found);
//have to get the actual applied data to find the actual length to skip because until then it can't compute the length due to the data type being dynamic
skipLen = program.getListing().getDataAt(found).getLength();
}
}
//If media does not apply correctly then it is not really a that media data type or there is other data in the way
catch (Exception e) {
// Not a valid embedded media or no room to apply it so just ignore it and skip it
}
}
// skip either the valid data that was found or skip one byte
// then do the next search
try {
start = found.add(skipLen);
found = memory.findBytes(start, end, mediaBytes, mask, true, monitor);
}
catch (AddressOutOfBoundsException e) {
// If media was at the very end of the address space, we will end up here
break;
}
}
private void addByteSearchPattern(MemoryBytePatternSearcher searcher, Program program,
List<Address> foundMedia, DataType mediaDT, String mediaName, byte[] bytes,
byte[] mask) {
if (bytes == null) {
return;
}
return foundMediaAddresses;
GenericMatchAction<DataType> action = new GenericMatchAction<DataType>(mediaDT) {
@Override
public void apply(Program prog, Address addr, Match match) {
//See if it is already an applied media data type
if (!program.getListing().isUndefined(addr, addr)) {
return;
}
try {
CreateDataCmd cmd = new CreateDataCmd(addr, mediaDT);
if (cmd.applyTo(program)) {
if (createBookmarksEnabled) {
program.getBookmarkManager().setBookmark(addr, BookmarkType.ANALYSIS,
"Embedded Media", "Found " + mediaName + " Embedded Media");
}
foundMedia.add(addr);
}
}
//If media does not apply correctly then it is not really a that media data type or there is other data in the way
catch (Exception e) {
// Not a valid embedded media or no room to apply it so just ignore it and skip it
}
}
};
GenericByteSequencePattern<DataType> genericByteMatchPattern =
new GenericByteSequencePattern<DataType>(bytes, mask, action);
searcher.addPattern(genericByteMatchPattern);
}
@Override

82
Ghidra/Features/Base/src/main/java/ghidra/program/util/ProgramMemoryUtil.java
View file

@ -177,8 +177,7 @@ public class ProgramMemoryUtil {
MemoryBlock[] tmpBlocks = new MemoryBlock[blocks.length];
int j = 0;
for (MemoryBlock block : blocks) {
if ((block.isInitialized() && withBytes) ||
(!block.isInitialized() && !withBytes)) {
if ((block.isInitialized() && withBytes) || (!block.isInitialized() && !withBytes)) {
tmpBlocks[j++] = block;
}
}
@ -493,6 +492,30 @@ public class ProgramMemoryUtil {
monitor = TaskMonitorAdapter.DUMMY_MONITOR;
}
byte[] addressBytes = getDirectAddressBytes(program, toAddress);
byte[] shiftedAddressBytes = getShiftedDirectAddressBytes(program, toAddress);
Memory memory = program.getMemory();
Set<Address> dirRefsAddrs = new TreeSet<>();
findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
}
return dirRefsAddrs;
}
/**
* Get a representation of an address as it would appear in bytes in memory.
*
* @param program program
* @param toAddress target address
* @return byte representation of toAddress
*/
public static byte[] getDirectAddressBytes(Program program, Address toAddress) {
Memory memory = program.getMemory();
boolean isBigEndian = memory.isBigEndian();
@ -536,6 +559,31 @@ public class ProgramMemoryUtil {
addressBytes, 0, addressBytes.length);
}
return addressBytes;
}
/**
* returns shifted address bytes if they are different than un-shifted
*
* @param program program
* @param toAddress target address
* @return shifted bytes, null if same as un-shifted
*/
public static byte[] getShiftedDirectAddressBytes(Program program, Address toAddress) {
byte[] addressBytes = getDirectAddressBytes(program, toAddress);
Memory memory = program.getMemory();
boolean isBigEndian = memory.isBigEndian();
DataConverter dataConverter;
if (isBigEndian) {
dataConverter = new BigEndianDataConverter();
}
else {
dataConverter = new LittleEndianDataConverter();
}
byte[] shiftedAddressBytes = null;
DataTypeManager dataTypeManager = program.getDataTypeManager();
DataOrganization dataOrganization = dataTypeManager.getDataOrganization();
@ -554,24 +602,23 @@ public class ProgramMemoryUtil {
}
}
// don't need this anymore - finding all 16 bit addrs in whole prog
// AddressRange segmentRange = null;
// if (toAddress instanceof SegmentedAddress) {
// // Restrict search to currentSegment range
// SegmentedAddressSpace segSpace = (SegmentedAddressSpace) toAddress.getAddressSpace();
// segmentRange =
// new AddressRangeImpl(segSpace.getAddress(currentSegment, 0), segSpace.getAddress(
// currentSegment, 0xffff));
// }
return shiftedAddressBytes;
}
Set<Address> dirRefsAddrs = new TreeSet<>();
findBytePattern(memory, blocks, addressBytes, alignment, dirRefsAddrs, monitor);
public static byte[] getImageBaseOffsets32Bytes(Program program, int alignment,
Address toAddress) {
if (shiftedAddressBytes != null) { // assume shifted address not supported with segmented memory
findBytePattern(memory, blocks, shiftedAddressBytes, alignment, dirRefsAddrs, monitor);
Address imageBase = program.getImageBase();
long offsetValue = toAddress.subtract(imageBase);
int offsetSize = 4; // 32 bit offset
byte[] bytes = new byte[offsetSize];
for (int i = 0; i < offsetSize; i++) {
bytes[i] = (byte) offsetValue;
offsetValue >>= 8; // Shift by a single byte.
}
return dirRefsAddrs;
return bytes;
}
/**
@ -627,8 +674,7 @@ public class ProgramMemoryUtil {
if (!block.isInitialized()) {
continue;
}
if (memoryRange != null &&
!memoryRange.intersects(block.getStart(), block.getEnd())) {
if (memoryRange != null && !memoryRange.intersects(block.getStart(), block.getEnd())) {
// skip blocks which do not correspond to currentSeg
continue;
}

82
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/AlignRule.java Normal file
View file

@ -0,0 +1,82 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
* alignment at an offset from the location the pattern matches.
*
* The pattern can be constructed or restored from XML of the form,
* where alignOffset=mark, alignmask=bits
*
* <align mark="0" bits="1"/>
*
*/
public class AlignRule implements PostRule {
private int alignOffset; // Position, relative to start of pattern, to check alignment at
private int alignmask; // Mask of bits that must be zero
public AlignRule() {
}
/**
* ByteSearch post search rule when a pattern is found. Used when a pattern must have a certain
* alignment at an offset from the location the pattern matches. The alignment is
* specified by the alignmask bits that must be zero.
*
* Normally alignOffset is 0, since most patterns will match at the address that must be aligned
* To align a match, use the following
*
* align to 2 = alignmask 0x1 - lower bit must be zero
* align to 4 = alignmask 0x3 - lower two bits must be zero
* align to 8 = alignmask 0x7 - lower three bits must be zero
* align to 16 = alignmask 0xF - lower four bits must be zero
* ....
* Other strange alignments could be specified, but most likely the above suffice.
* @param alignOffset - bytes offset from pattern to check for alignment
* @param alignmask - the mask where a 1 bit must be zero
*/
public AlignRule(int alignOffset, int alignmask) {
this.alignOffset = alignOffset;
this.alignmask = alignmask;
}
@Override
public boolean apply(Pattern pat, long matchoffset) {
int off = (int) matchoffset;
return (((off + alignOffset) & alignmask) == 0);
}
@Override
public void restoreXml(XmlPullParser parser) {
XmlElement el = parser.start("align");
alignOffset = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
alignmask = (1 << bits) - 1;
parser.end();
}
public int getAlignMask() {
return alignmask;
}
}

118
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DittedBitSequence.java
View file

@ -21,6 +21,19 @@ import java.util.zip.CRC32;
import ghidra.xml.XmlPullParser;
/**
* A pattern of bits/mask to match to a stream of bytes. The bits/mask can be of any length.
* The sequence can be initialized by:
*
* a string
* an array of bytes (no mask)
* an array of bytes and for mask
*
* The dits represent bits(binary) or nibbles(hex) that are don't care, for example:
* 0x..d.4de2 ....0000 .1...... 00101101 11101001
* where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
*/
public class DittedBitSequence {
//Given a byte 0-255 (NOT a signed byte), retrieves its popcount.
@ -42,7 +55,7 @@ public class DittedBitSequence {
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 //240-255
};
protected int index; // Unique index assigned to this sequence
private int index; // Unique index assigned to this sequence
private byte[] bits; // value bits contained in the sequence
private byte[] dits; // a 1 indicates the bit is not ditted
@ -53,7 +66,9 @@ public class DittedBitSequence {
/**
* Constructor from a ditted-bit-sequence string where white space is ignored (e.g., "10..11.0");
* @param dittedBitData
*
* @param dittedBitData ditted sequence specified as a string
*
* @throws IllegalArgumentException if invalid dittedBitData specified
*/
public DittedBitSequence(String dittedBitData) {
@ -64,8 +79,8 @@ public class DittedBitSequence {
* Constructor from a ditted-bit string where white space is ignored. If there are no dits,
* {@code hex} is true, and {@code hex} does not begin with {code 0x}, {@code 0x} will be
* prepended to the string before constructing the {@link DittedBitSequence}.
* @param dittedBitData
* @param hex
* @param dittedBitData string of bits and dits or hex numbers and dits (e.g., 0.1..0, 0xAB..)
* @param hex true to force hex on the sequence
*/
public DittedBitSequence(String dittedBitData, boolean hex) {
if (hex && !dittedBitData.contains(".")) {
@ -101,7 +116,8 @@ public class DittedBitSequence {
/**
* Construct a sequence of bytes to search for. No bits are masked off.
* @param bytes
*
* @param bytes byte values that must match
*/
public DittedBitSequence(byte[] bytes) {
bits = bytes;
@ -164,22 +180,40 @@ public class DittedBitSequence {
return true;
}
public DittedBitSequence concatenate(DittedBitSequence op2) {
/**
* Concatenates a sequence to the end of another sequence and
* returns a new sequence.
*
* @param toConat sequence to concatenate to this sequence
*
* @return a new sequence that is the concat of this and toConcat
*/
public DittedBitSequence concatenate(DittedBitSequence toConat) {
DittedBitSequence res = new DittedBitSequence();
res.bits = new byte[bits.length + op2.bits.length];
res.bits = new byte[bits.length + toConat.bits.length];
res.dits = new byte[res.bits.length];
for (int i = 0; i < bits.length; ++i) {
res.bits[i] = bits[i];
res.dits[i] = dits[i];
}
for (int i = 0; i < op2.bits.length; ++i) {
res.bits[bits.length + i] = op2.bits[i];
res.dits[bits.length + i] = op2.dits[i];
for (int i = 0; i < toConat.bits.length; ++i) {
res.bits[bits.length + i] = toConat.bits[i];
res.dits[bits.length + i] = toConat.dits[i];
}
return res;
}
/**
* Check for a match of a value at a certain offset in the pattern.
* An outside matcher will keep track of the match position within this
* ditted bit sequence. Then call this method to match.
*
* @param pos position in the pattern to match
* @param val a byte to be match at the given byte offset in the pattern
*
* @return true if the byte matches the sequence mask/value
*/
public boolean isMatch(int pos, int val) {
if (pos >= bits.length) {
return false;
@ -187,14 +221,38 @@ public class DittedBitSequence {
return ((byte) (val & dits[pos])) == bits[pos];
}
/**
* Set a an index in a larger sequence, or identifing id on this pattern
*
* @param index - index in match sequence, or unique id
*/
public void setIndex(int index) {
this.index = index;
}
/**
* Get the index or identifying id attached to this pattern
*
* @return index or unique id attached to this sequence
*/
public int getIndex() {
return index;
}
/**
* get the size of this sequence in bytes
*
* @return size in bytes
*/
public int getSize() {
return bits.length;
}
/**
* Get number of bits that must be 0/1
*
* @return number of bits that are not don't care (ditted)
*/
public int getNumFixedBits() {
int popcnt = 0;
for (byte dit : dits) {
@ -203,7 +261,11 @@ public class DittedBitSequence {
return popcnt;
}
//Return the number of dits.
/**
* Get number of bits that are ditted (don't care)
*
* @return number of ditted bits (don't care)
*/
public int getNumUncertainBits() {
int popcnt = 0;
for (byte dit : dits) {
@ -235,6 +297,11 @@ public class DittedBitSequence {
return buf.toString();
}
/**
* get a ditted hex string representing this sequence
*
* @return ditted hex string
*/
public String getHexString() {
String uncompressed = this.toString();
String[] parts = uncompressed.trim().split(" ");
@ -260,6 +327,17 @@ public class DittedBitSequence {
return sb.toString();
}
/**
* restore ditted string from XML stream with hex/binary ditted sequences in the form:
* <data> 0x..d.4de2 ....0000 .1...... 00101101 11101001 </data>
* where 0x starts a hex number and '.' is a don't care nibble (hex) or bit (binary)
*
* @param parser XML pull parser stream
*
* @return number of bytes read from XML <data> tag
*
* @throws IOException if XML read has an error
*/
protected int restoreXmlData(XmlPullParser parser) throws IOException {
parser.start("data");
String text = parser.end().getText();
@ -272,6 +350,16 @@ public class DittedBitSequence {
}
}
/**
* Initialize this sequence with a ditted sequence from a string in the form
* (e.g. - 011...1., 0x.F, 01110011 0xAB)
*
* @param text ditted sequence
*
* @return number of bytes in the ditted sequence
*
* @throws IllegalArgumentException if string is malformed
*/
private int initFromDittedStringData(String text) throws IllegalArgumentException {
int markOffset = -1;
int mode = -1; // -1: looking for start, -2: skip to EOL, 0: hex mode, 1: binary mode
@ -372,6 +460,13 @@ public class DittedBitSequence {
return markOffset;
}
/**
* Get the number of bits that are fixed, not ditted (don't care)
*
* @param marked number of bytes in the pattern to check
*
* @return number of initial fixed bits
*/
public int getNumInitialFixedBits(int marked) {
if (dits == null) {
return 0;
@ -385,5 +480,4 @@ public class DittedBitSequence {
}
return popcnt;
}
}

5
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/DummyMatchAction.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -20,6 +19,9 @@ import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Program;
import ghidra.xml.XmlPullParser;
/**
* Dummy action attached to a match sequence. Action is not restored from XML
*/
public class DummyMatchAction implements MatchAction {
@Override
@ -32,4 +34,3 @@ public class DummyMatchAction implements MatchAction {
}
}

61
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericByteSequencePattern.java Normal file
View file

@ -0,0 +1,61 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import ghidra.program.model.data.DataType;
/**
* Templated simple DittedBitSequence Pattern for a byte/mask pattern and associated action.
* The DittedBitSequence is provided by value and mask in byte arrays.
*
* This class is normally used to find some number of SequencePatterns within a seqence of bytes.
* When the byte/mask pattern is matched, the GenericMatchAction will be "applied".
*
* @param <T> the class of match action, used to specify a specialized momento to be used by the action when it is "applied".
*/
public class GenericByteSequencePattern<T> extends Pattern {
/**
* Construct a sequence of bytes with no mask, and associated action
* to be called if this pattern matches.
*
* @param bytesSequence sequence of bytes to match
* @param action action to apply if the match succeeds
*/
public GenericByteSequencePattern(byte[] bytesSequence, GenericMatchAction<T> action) {
super(new DittedBitSequence(bytesSequence), 0, new PostRule[0], new MatchAction[1]);
MatchAction[] matchActions = getMatchActions();
matchActions[0] = action;
}
/**
* Construct a sequence of bytes with a mask, and associated action
* to be called if this pattern matches.
*
* @param bytesSequence sequence of bytes to match
* @param mask mask, bits that are 1 must match the byteSequence bits
* @param action to apply if the match succeeds
*/
public GenericByteSequencePattern(byte[] bytesSequence, byte[] mask,
GenericMatchAction<DataType> action) {
super(new DittedBitSequence(bytesSequence, mask), 0, new PostRule[0], new MatchAction[1]);
MatchAction[] matchActions = getMatchActions();
matchActions[0] = action;
}
}

43
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/GenericMatchAction.java Normal file
View file

@ -0,0 +1,43 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
/**
* Template for generic match action attached to a match sequence.
* Used to store an associated value to the matching sequence.
* The associated value can be retrieved when the sequence is matched.
*
* @param <T> - object to attach to match sequence, generally used to specify
* a specialized momento to be used by the action when it is "applied".
*/
public class GenericMatchAction<T> extends DummyMatchAction {
T matchValue;
/**
* Construct a match action used when a match occurs for some GenericByteSequece
* @param matchValue specialized object used when match occurs
*/
public GenericMatchAction(T matchValue) {
this.matchValue = matchValue;
}
/**
* @return the specialized object associated with this match action
*/
public T getMatchValue() {
return this.matchValue;
}
}

56
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Match.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Match.java
View file

@ -15,13 +15,27 @@
*/
package ghidra.util.bytesearch;
/**
* Represents a match of a DittedBitSequence at a given offset in a byte sequence.
*
* There is a hidden assumption that the sequence is actually a Pattern
* that might have a ditted-bit-sequence, a set of match actions,
* and post match rules/checks
*
*/
public class Match {
private DittedBitSequence sequence; // Pattern that matches
private long offset; // starting offset within bytestream of match
private DittedBitSequence sequence; // Pattern that matched
private long offset; // Offset within bytestream where the match occurred
public Match(DittedBitSequence seq, long off) {
sequence = seq;
offset = off;
/**
* Construct a Match of a DittedBitSequence at an offset within a byte stream.
* Object normally used when a match occurs during a MemoryBytePatternSearch.
* @param sequence that matched
* @param offset from the start of byte stream where the matched occured
*/
public Match(DittedBitSequence sequence, long offset) {
this.sequence = sequence;
this.offset = offset;
}
/**
@ -39,42 +53,70 @@ public class Match {
return sequence.getNumFixedBits() - sequence.getNumInitialFixedBits(marked);
}
/**
* @return actions associated with this match
*/
public MatchAction[] getMatchActions() {
return ((Pattern) sequence).getMatchActions();
}
/**
* @return size in bytes of sequence
*/
public int getSequenceSize() {
return sequence.getSize();
}
/**
* @return index of sequence in a possibly longer set of sequences
*/
public int getSequenceIndex() {
return sequence.getIndex();
}
/**
* @return the offset of the match within a longer byte sequence
*/
public long getMarkOffset() {
return offset + ((Pattern) sequence).getMarkOffset();
}
/**
* @return offset of match in sequence of bytes
*/
public long getMatchStart() {
return offset;
}
/**
* Check that the possible post rules are satisfied
*
* @param streamoffset offset within from match location to check postrules.
*
* @return true if post rules are satisfied
*/
public boolean checkPostRules(long streamoffset) {
long curoffset = streamoffset + offset;
Pattern pattern = (Pattern) sequence;
PostRule[] postRules = pattern.getPostRules();
for (int i = 0; i < postRules.length; ++i) {
if (!postRules[i].apply(pattern, curoffset)) {
for (PostRule postRule : postRules) {
if (!postRule.apply(pattern, curoffset)) {
return false;
}
}
return true;
}
/**
* @return ditted bit sequence as a string
*/
public String getHexString() {
return sequence.getHexString();
}
/**
* @return the sequence that was matched
*/
public DittedBitSequence getSequence() {
return sequence;
}

18
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/MatchAction.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MatchAction.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,11 +20,22 @@ import ghidra.program.model.listing.Program;
import ghidra.xml.XmlPullParser;
/**
* Action that should be applied to a Program at the Address a pattern matches
*
* Interface for a match action to be taken for the Program@Address for a ditted bit seqence pattern
*/
public interface MatchAction {
public void apply(Program program,Address addr,Match match);
/**
* Apply the match action to the program at the address.
*
* @param program program in which the match occurred
* @param addr where the match occured
* @param match information about the match that occurred
*/
public void apply(Program program, Address addr, Match match);
/**
* Action can be constructed from XML
*
* @param parser XML pull parser to restore action from XML
*/
public void restoreXml(XmlPullParser parser);
}

268
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/MemoryBytePatternSearcher.java Normal file
View file

@ -0,0 +1,268 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import ghidra.program.model.address.*;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* Multi pattern/mask/action memory searcher
* Patterns must be supplied/added, or a pre-initialized searchState supplied
*
* Preload search patterns and actions, then call search method.
*/
public class MemoryBytePatternSearcher {
private static final long RESTRICTED_PATTERN_BYTE_RANGE = 32;
SequenceSearchState root = null;
ArrayList<Pattern> patternList;
private String searchName = "";
private boolean doExecutableBlocksOnly = false; // only search executable blocks
private long numToSearch = 0;
private long numSearched = 0;
/**
* Create with pre-created patternList
* @param searchName name of search
* @param patternList - list of patterns(bytes/mask/action)
*/
public MemoryBytePatternSearcher(String searchName, ArrayList<Pattern> patternList) {
this.searchName = searchName;
this.patternList = new ArrayList<Pattern>(patternList);
}
/**
* Create with an initialized SequenceSearchState
* @param searchName name of search
* @param root search state pre-initialized
*/
public MemoryBytePatternSearcher(String searchName, SequenceSearchState root) {
this.searchName = searchName;
this.root = root;
}
/**
* Create with no patternList, must add patterns before searching
* @param searchName name of search
*
*/
public MemoryBytePatternSearcher(String searchName) {
this.searchName = searchName;
patternList = new ArrayList<>();
}
/**
* Add a search pattern
* @param pattern - pattern(bytes/mask/action)
*/
public void addPattern(Pattern pattern) {
patternList.add(pattern);
}
public void setSearchExecutableOnly(boolean doExecutableBlocksOnly) {
this.doExecutableBlocksOnly = doExecutableBlocksOnly;
}
/**
* Search initialized memory blocks for all patterns(bytes/mask/action).
* Call associated action for each pattern matched.
*
* @param program to be searched
* @param searchSet set of bytes to restrict search, if null or empty then search all memory blocks
* @param monitor allow canceling and reporting of progress
*
* @throws CancelledException if canceled
*/
public void search(Program program, AddressSetView searchSet, TaskMonitor monitor)
throws CancelledException {
if (root == null) {
root = SequenceSearchState.buildStateMachine(patternList);
}
numToSearch = getNumToSearch(program, searchSet);
monitor.setMessage(searchName + " Search");
monitor.initialize(numToSearch);
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block : blocks) {
monitor.setProgress(numSearched);
// check if entire block has anything that is searchable
if (!block.isInitialized()) {
continue;
}
if (doExecutableBlocksOnly && !block.isExecute()) {
continue;
}
if (searchSet != null && !searchSet.isEmpty() &&
!searchSet.intersects(block.getStart(), block.getEnd())) {
continue;
}
try {
searchBlock(root, program, block, searchSet, monitor);
}
catch (IOException e) {
Msg.error(this, "Unable to scan block " + block.getName() + " for " + searchName);
}
numSearched += block.getSize();
}
}
private long getNumToSearch(Program program, AddressSetView searchSet) {
long numAddresses = 0;
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block : blocks) {
// check if entire block has anything that is searchable
if (!block.isInitialized()) {
continue;
}
if (doExecutableBlocksOnly && !block.isExecute()) {
continue;
}
if (searchSet != null && !searchSet.isEmpty() &&
!searchSet.intersects(block.getStart(), block.getEnd())) {
continue;
}
numAddresses += block.getSize();
}
return numAddresses;
}
/**
* Search through bytes of a memory block using the finite state machine -root-
* Apply any additional rules for matching patterns.
*
* @param program is the Program being searched
* @param block is the specific block of bytes being searched
*
* @throws IOException exception during read of memory
* @throws CancelledException canceled search
*/
private void searchBlock(SequenceSearchState rootState, Program program, MemoryBlock block,
AddressSetView restrictSet, TaskMonitor monitor)
throws IOException, CancelledException {
// if no restricted set, make restrict set the full block
AddressSet doneSet;
if (restrictSet == null || restrictSet.isEmpty()) {
doneSet = new AddressSet(block.getStart(), block.getEnd());
}
else {
doneSet = restrictSet.intersectRange(block.getStart(), block.getEnd());
}
long numInDoneSet = doneSet.getNumAddresses();
long numInBlock = block.getSize();
Address blockStartAddr = block.getStart();
// pull each range off the restricted set
long progress = monitor.getProgress();
AddressRangeIterator addressRanges = doneSet.getAddressRanges();
long numDone = 0;
while (addressRanges.hasNext()) {
monitor.checkCanceled();
monitor.setMessage(searchName + " Search");
monitor.setProgress(progress + (long) (numInBlock * ((float) numDone / numInDoneSet)));
AddressRange addressRange = addressRanges.next();
long numAddressesInRange = addressRange.getLength();
ArrayList<Match> mymatches = new ArrayList<>();
long streamoffset = blockStartAddr.getOffset();
// Give block a starting/ending point before this address to search
// patterns might start before, since they have a pre-pattern
// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
// normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
// normally the whole memory blocks will get searched.
long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
if (blockOffset <= 0) {
// don't go before the block start
blockOffset = 0;
}
// compute number of bytes in the range + 1, and don't search more than that.
long maxBlockSearchLength =
addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
InputStream data = block.getData();
data.skip(blockOffset);
rootState.apply(data, maxBlockSearchLength, mymatches, monitor);
monitor.checkCanceled();
monitor.setMessage(searchName + " (Examine Matches)");
// TODO: DANGER there is much offset<-->address calculation here
// should be OK, since they are all relative to the block.
long matchProgress = progress + (long) (numInBlock * ((float) numDone / numInDoneSet));
for (int i = 0; i < mymatches.size(); ++i) {
monitor.checkCanceled();
monitor.setProgress(
matchProgress + (long) (numAddressesInRange * ((float) i / mymatches.size())));
Match match = mymatches.get(i);
Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
if (!match.checkPostRules(streamoffset + blockOffset)) {
continue;
}
MatchAction[] matchactions = match.getMatchActions();
preMatchApply(matchactions, addr);
for (MatchAction matchaction : matchactions) {
matchaction.apply(program, addr, match);
}
postMatchApply(matchactions, addr);
}
numDone += numAddressesInRange;
}
}
/**
* Called before any match rules are applied
* @param matchactions actions that matched
* @param addr address of match
*/
public void preMatchApply(MatchAction[] matchactions, Address addr) {
// override if any initialization needs to be done before rule application
}
/**
* Called after any match rules are applied
* Can use for cross post rule matching state application and cleanup.
* @param matchactions actions that matched
* @param addr adress of match
*/
public void postMatchApply(MatchAction[] matchactions, Address addr) {
// override if any cleanup from rule match application is needed
}
}

24
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/Pattern.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/Pattern.java
View file

@ -24,12 +24,22 @@ import generic.jar.ResourceFile;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.*;
/**
* Pattern is an association of a DittedBitSequence to match,
* a set of post rules after a match is found that must be satisfied,
* and a set of actions to be taken if the pattern matches.
*
* These patterns can be restored from an XML file.
*/
public class Pattern extends DittedBitSequence {
private int markOffset; // Within pattern what is the 'marked' byte
private PostRule[] postrule;
private MatchAction[] actions;
/**
* Construct an empty pattern. Use XML to initialize
*/
public Pattern() {
markOffset = 0;
postrule = null;
@ -37,7 +47,17 @@ public class Pattern extends DittedBitSequence {
}
public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray, MatchAction[] matchArray) {
/**
* Construct the pattern based on a DittedByteSequence a match offset, post matching rules,
* and a set of actions to take when the match occurs.
*
* @param seq DittedByteSequence
* @param offset offset from the actual match location to report a match
* @param postArray post set of rules to check for the match
* @param matchArray MatchActions to apply when a match occurs
*/
public Pattern(DittedBitSequence seq, int offset, PostRule[] postArray,
MatchAction[] matchArray) {
super(seq);
markOffset = offset;
postrule = postArray;
@ -52,7 +72,7 @@ public class Pattern extends DittedBitSequence {
return actions;
}
public void setMatchActions(MatchAction[] actions){
public void setMatchActions(MatchAction[] actions) {
this.actions = actions;
}

15
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternFactory.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternFactory.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +15,22 @@
*/
package ghidra.util.bytesearch;
/**
* Interface for factories that create Match Pattern classes
*/
public interface PatternFactory {
/**
* Get a named match action
*
* @param nm name of action to find
* @return match action with the given name, null otherwise
*/
public MatchAction getMatchActionByName(String nm);
/**
* Get a named post match rule by name
* @param nm name of the post rule
* @return the post rule with the name, null otherwise
*/
public PostRule getPostRuleByName(String nm);
}

68
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PatternPairSet.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PatternPairSet.java
View file

@ -23,10 +23,35 @@ import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
/**
* Two collections of patterns that are paired together to create larger patterns
* The final large patterns all must first match a pattern from the "pre" pattern collection
* followed immediately by a pattern from the "post" pattern collection
* A set of "pre" DittedBitSequences and a set of "post" Patterns are paired to form a larger pattern.
* To match, a sequence from the "pre" sequence set must first match, then one of the "post" patterns
* is matched relative to the matching "pre" pattern. This class is really a storage object for the
* patterns and provides a mechanism to read the pre/post patterns from an XML file.
*
* The larger pattern has the idea of bits of check, which means the number of bits that are fixed to
* a value when matching (not don't care). There is a pre pattern bits of check and post pattern bits
* of check. The bits of check are used to statistically gauge the accuracy of the pattern.
*
* An example of the XML format follows:
* <patternpairs totalbits="32" postbits="16">
* <prepatterns>
* <data>0xe12fff1. </data>
* <data>0xe12fff1e 0x46c0 </data>
* <data>0xe12fff1e 0xe1a00000 </data>
* </prepatterns>
*
* <postpatterns>
* <data> 0xe24dd... 11101001 00101101 .1...... ....0000 </data>
* <data> 11101001 00101101 .1...... ....0000 0xe24dd... </data>
* <data> 11101001 00101101 .1...... ....0000 0x........ 0xe24dd... </data>
* <align mark="0" bits="3"/>
* <setcontext name="TMode" value="0"/>
* <funcstart/>
* </postpatterns>
* </patternpairs>
*
* Note: The post Patterns can also have a set of rules that must be satisfied along with one of the
* Pattern DittedBitSequence matches.
*/
public class PatternPairSet {
private int totalBitsOfCheck; // Minimum number of bits of check in final patterns
@ -34,55 +59,68 @@ public class PatternPairSet {
private ArrayList<DittedBitSequence> preSequences;
private ArrayList<Pattern> postPatterns;
/**
* Construct an empty PatternPairSet. Use XML to initialize the pattern sets.
*/
public PatternPairSet() {
preSequences = new ArrayList<DittedBitSequence>();
postPatterns = new ArrayList<Pattern>();
}
public void createFinalPatterns(ArrayList<Pattern> finalpats) {
for(int i=0;i<postPatterns.size();++i) {
for (int i = 0; i < postPatterns.size(); ++i) {
Pattern postpattern = postPatterns.get(i);
int postcheck = postpattern.getNumFixedBits();
if (postcheck < postBitsOfCheck) {
continue;
}
for(int j=0;j<preSequences.size();++j) {
DittedBitSequence prepattern = preSequences.get(j);
for (DittedBitSequence prepattern : preSequences) {
int precheck = prepattern.getNumFixedBits();
if (precheck + postcheck < totalBitsOfCheck) {
continue;
}
DittedBitSequence concat = prepattern.concatenate(postpattern);
Pattern finalpattern = new Pattern(concat,prepattern.getSize(),postpattern.getPostRules(),postpattern.getMatchActions());
Pattern finalpattern = new Pattern(concat, prepattern.getSize(),
postpattern.getPostRules(), postpattern.getMatchActions());
finalpats.add(finalpattern);
}
}
}
/**
* Add this PatternPairSets post patterns to an existing arraylist of patterns.
* @param postpats array to add this PatternPairSets post patterns into
*/
public void extractPostPatterns(ArrayList<Pattern> postpats) {
for(int i=0;i<postPatterns.size();++i) {
for (int i = 0; i < postPatterns.size(); ++i) {
postpats.add(postPatterns.get(i));
}
}
public void restoreXml(XmlPullParser parser,PatternFactory pfactory) throws IOException {
/**
* Restore PatternPairSet from XML pull parser
* @param parser XML pull parser
* @param pfactory pattern factory user to construct patterns
* @throws IOException if pull parsing fails
*/
public void restoreXml(XmlPullParser parser, PatternFactory pfactory) throws IOException {
XmlElement el = parser.start("patternpairs");
totalBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("totalbits"));
postBitsOfCheck = SpecXmlUtils.decodeInt(el.getAttribute("postbits"));
parser.start("prepatterns");
el= parser.peek();
while(el.isStart()) {
el = parser.peek();
while (el.isStart()) {
DittedBitSequence preseq = new DittedBitSequence();
preseq.restoreXmlData(parser);
preSequences.add(preseq);
el = parser.peek();
}
parser.end();
while(parser.peek().isStart()) {
while (parser.peek().isStart()) {
parser.start("postpatterns");
el = parser.peek();
ArrayList<DittedBitSequence> postdit = new ArrayList<DittedBitSequence>();
while(el.isStart() && el.getName().equals("data")) {
while (el.isStart() && el.getName().equals("data")) {
DittedBitSequence postseq = new DittedBitSequence();
postseq.restoreXmlData(parser);
if (postseq.getNumFixedBits() >= postBitsOfCheck) {
@ -99,8 +137,8 @@ public class PatternPairSet {
postRuleArray.toArray(postRules);
MatchAction[] matchActions = new MatchAction[matchActionArray.size()];
matchActionArray.toArray(matchActions);
for(int i=0;i<postdit.size();++i) {
Pattern postpat = new Pattern(postdit.get(i),0,postRules,matchActions);
for (DittedBitSequence element : postdit) {
Pattern postpat = new Pattern(element, 0, postRules, matchActions);
postPatterns.add(postpat);
}
parser.end(); // End postpatterns

17
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/PostRule.java → Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/PostRule.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -18,8 +17,22 @@ package ghidra.util.bytesearch;
import ghidra.xml.XmlPullParser;
/**
* Inteface for post match rules that are checked after a match is idenfied
*/
public interface PostRule {
public boolean apply(Pattern pat,long matchoffset);
/**
* Apply a post rule given the matching pattern and offset into the byte stream.
* @param pat pattern that matched
* @param matchoffset offset of the match
* @return true if the PostRule is satisfied
*/
public boolean apply(Pattern pat, long matchoffset);
/**
* Can restore state of instance PostRule from XML
*
* @param parser XML pull parser
*/
public void restoreXml(XmlPullParser parser);
}

466
Ghidra/Features/Base/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java Normal file
View file

@ -0,0 +1,466 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import ghidra.util.task.TaskMonitor;
/**
* SeqenceSearchState holds the state of a search for a DittedBitSequence within a byte
* sequence.
*/
public class SequenceSearchState implements Comparable<SequenceSearchState> {
private static final int PATTERN_ENDED = Integer.MAX_VALUE;
private SequenceSearchState parent;
private ArrayList<DittedBitSequence> possible; // Patterns that could still match in this state
private ArrayList<DittedBitSequence> success; // Patterns that have matched successfully if we reached this state
private SequenceSearchState[] trans; // State transitions based on next byte
/**
* Construct a sub sequence state with a parent sequence
*
* @param parent parent SequenceSearchState
*/
public SequenceSearchState(SequenceSearchState parent) {
this.parent = parent;
possible = new ArrayList<DittedBitSequence>();
success = null;
trans = null;
}
/**
* @return maximum number of bytes that could be matched by this sequence
*/
public int getMaxSequenceSize() {
int max = 0;
for (DittedBitSequence element : possible) {
int val = element.getSize();
if (val > max) {
max = val;
}
}
return max;
}
/**
* Add a pattern to this search sequence. The last pattern added is the successful
* match pattern.
* @param pat pattern to add
* @param pos position within the current set of patterns to add this pattern
*/
public void addSequence(DittedBitSequence pat, int pos) {
possible.add(pat);
if (pos == pat.getSize()) {
if (success == null) {
success = new ArrayList<DittedBitSequence>();
}
success.add(pat);
}
}
/**
* Sort the sequences that have been added
*/
public void sortSequences() {
Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
@Override
public int compare(DittedBitSequence o1, DittedBitSequence o2) {
return o1.getIndex() - o2.getIndex();
}
};
Collections.sort(possible, comp);
if (success != null) {
Collections.sort(success, comp);
}
}
@Override
public int compareTo(SequenceSearchState o) {
int i = 0;
for (;;) {
if (possible.size() <= i) {
if (o.possible.size() <= i) {
return 0;
}
return -1;
}
if (o.possible.size() <= i) {
return 1;
}
int indus = possible.get(i).getIndex(); // Lexicographic compare an sequence of sequences
int indthem = o.possible.get(i).getIndex();
if (indus != indthem) {
return (indus < indthem) ? -1 : 1;
}
i += 1;
}
}
private void buildSingleTransition(ArrayList<SequenceSearchState> all, int pos, int val) {
SequenceSearchState newstate = null;
for (DittedBitSequence curpat : possible) {
if (curpat.isMatch(pos, val)) {
if (newstate == null) {
newstate = new SequenceSearchState(this);
}
newstate.addSequence(curpat, pos + 1);
}
}
trans[val] = newstate;
if (newstate != null) {
newstate.sortSequences();
all.add(newstate);
}
}
private void exportSuccess(ArrayList<Match> match, int offset) {
for (DittedBitSequence succes : success) { // If we found matches
Match newmatch = new Match(succes, offset);
match.add(newmatch);
}
}
/**
* Merge in -op- and this as a single state
* @param op
*/
private void merge(SequenceSearchState op) {
SequenceSearchState parent = op.parent;
for (int i = 0; i < 256; ++i) {
if (parent.trans[i] == op) {
parent.trans[i] = this; // Should be replaced with this
}
}
if (op.success != null) { // Merge
if (success == null) {
success = op.success;
}
else {
ArrayList<DittedBitSequence> tmp = new ArrayList<DittedBitSequence>();
int i = 0;
int j = 0;
int curpat = -1;
int thispat = success.get(i).getIndex();
int oppat = op.success.get(j).getIndex();
while ((i < success.size()) || (j < op.success.size())) {
if (thispat == oppat) {
if (curpat != thispat) {
tmp.add(success.get(i));
curpat = thispat;
}
i += 1;
j += 1;
thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
oppat =
(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
}
else if (thispat < oppat) {
if (curpat != thispat) {
tmp.add(success.get(i));
curpat = thispat;
}
i += 1;
thispat = (i == success.size()) ? PATTERN_ENDED : success.get(i).getIndex();
}
else {
if (curpat != oppat) {
tmp.add(op.success.get(j));
curpat = oppat;
}
j += 1;
oppat =
(j == op.success.size()) ? PATTERN_ENDED : op.success.get(j).getIndex();
}
}
success = tmp;
}
}
}
/**
* Try to match this Sequence to the byteArray, and add any matches to the match list
* @param bytearray array of bytes to match
* @param numbytes retrict number of bytes to allow to match
* @param match list of matches, the result
*/
public void sequenceMatch(byte[] bytearray, int numbytes, ArrayList<Match> match) {
int subindex = 0;
SequenceSearchState curstate = this;
do {
if (curstate.success != null) {
curstate.exportSuccess(match, 0);
}
if (subindex >= numbytes) {
return;
}
curstate = curstate.trans[0xff & bytearray[subindex]]; // Perform state transition based on next byte in buffer
subindex += 1;
}
while (curstate != null);
}
/**
* Search for patterns in a byte array. All matches are returned.
* @param buffer is the array of bytes to search
* @param match is populated with a Match object for each pattern and position that matches
*/
public void apply(byte[] buffer, ArrayList<Match> match) {
SequenceSearchState curstate;
int subindex;
for (int offset = 0; offset < buffer.length; ++offset) {
curstate = this; // New starting offset -> Root state
subindex = offset;
do {
if (curstate.success != null) {
curstate.exportSuccess(match, offset);
}
if (subindex >= buffer.length) { // if we've run out of bytes, must restart at next offset
break;
}
curstate = curstate.trans[0xff & buffer[subindex]]; // Perform state transition based on next byte
subindex += 1;
}
while (curstate != null);
}
}
/**
* Search for pattern in the stream -in-.
* @param in - The stream to scan for matches
* @param match - Any matches are appended as Match records to this ArrayList
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in, ArrayList<Match> match, TaskMonitor monitor)
throws IOException {
apply(in, -1L, match, monitor);
}
/**
* Search for pattern in the stream -in-.
* @param in - The stream to scan for matches
* @param maxBytes - The maximum number of bytes to scan forward in this stream
* @param match - Any matches are appended as Match records to this ArrayList
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in, long maxBytes, ArrayList<Match> match, TaskMonitor monitor)
throws IOException {
long progress = monitor.getProgress();
int maxSize = getMaxSequenceSize() + 1;
if (maxSize < 4096) {
maxSize = 4096;
}
if (maxBytes > 0) {
maxBytes += getMaxSequenceSize() + 1;
}
byte[] firstBuf = new byte[maxSize];
byte[] secondBuf = new byte[maxSize];
byte[] curBuf;
SequenceSearchState curState;
int fullBuffers; // Number of buffers that are completely full
int ra = in.read(firstBuf);
if (ra == firstBuf.length) {
ra = in.read(secondBuf);
if (ra == secondBuf.length) {
fullBuffers = 2;
}
else {
if (ra < 0) {
ra = 0;
}
fullBuffers = 1;
byte[] tmp = new byte[ra];
for (int i = 0; i < ra; ++i) {
tmp[i] = secondBuf[i];
}
secondBuf = tmp;
}
}
else if (ra < 0) {
return; // No bytes at all were read
}
else {
byte[] tmp = new byte[ra];
for (int i = 0; i < ra; ++i) {
tmp[i] = firstBuf[i];
}
firstBuf = tmp;
fullBuffers = 0;
secondBuf = new byte[0];
}
int offset = 0;
int bufRelativeOffset = 0;
int subIndex;
while (fullBuffers == 2) {
curState = this; // New starting offset -> Root state
subIndex = bufRelativeOffset;
curBuf = firstBuf;
do {
if (curState.success != null) {
curState.exportSuccess(match, offset);
}
if (subIndex >= curBuf.length) { // check that we have enough bytes in current buffer
curBuf = secondBuf; // If not, switch to secondary buffer
subIndex = 0;
}
curState = curState.trans[0xff & curBuf[subIndex]]; // Perform state transition based on next byte in buffer
subIndex += 1;
}
while (curState != null);
offset += 1; // Advance to next starting offset
if (maxBytes > 0 && offset > maxBytes) {
break;
}
bufRelativeOffset += 1;
if (bufRelativeOffset == firstBuf.length) { // If starting offset no longer falls in firstbuf
byte[] tmp = firstBuf; // Switch firstbuf with secondbuf
firstBuf = secondBuf;
secondBuf = tmp;
ra = in.read(secondBuf); // refill secondbuf (old firstbuf) with new bytes
if (monitor != null) {
if (monitor.isCancelled()) {
return;
}
monitor.setProgress(progress + offset);
}
if (ra != secondBuf.length) {
fullBuffers = 1;
if (ra < 0) {
ra = 0;
}
tmp = new byte[ra];
for (int i = 0; i < ra; ++i) {
tmp[i] = secondBuf[i];
}
secondBuf = tmp;
}
bufRelativeOffset = 0;
}
}
while (fullBuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
if (secondBuf.length == 0) {
fullBuffers = 0;
}
curState = this;
subIndex = bufRelativeOffset;
curBuf = firstBuf;
do {
if (curState.success != null) {
curState.exportSuccess(match, offset);
}
if (subIndex >= curBuf.length) {
if (curBuf == secondBuf) {
break; // Out of data, all pending patterns fail
}
curBuf = secondBuf;
subIndex = 0;
if (curBuf.length == 0) {
break;
}
}
curState = curState.trans[0xff & curBuf[subIndex]];
subIndex += 1;
}
while (curState != null);
offset += 1;
bufRelativeOffset += 1;
if (bufRelativeOffset == firstBuf.length) {
if (fullBuffers == 0) {
break;
}
firstBuf = secondBuf;
fullBuffers = 0;
bufRelativeOffset = 0;
secondBuf = new byte[0];
}
}
}
/**
* Build a new transition level for the state machine
*
* @param prev previous search sequences
* @param pos position within the search sequence state for this level
* @return list of possible new search states to be added to the state machine
*/
static ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,
int pos) {
ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iterator = prev.iterator();
while (iterator.hasNext()) { // For each current state
SequenceSearchState next = iterator.next();
next.trans = new SequenceSearchState[256];
for (int i = 0; i < 256; ++i) { // Try every byte transition
next.buildSingleTransition(res, pos, i);
}
}
if (res.isEmpty()) {
return res;
}
// Prepare to dedup the states
Collections.sort(res);
ArrayList<SequenceSearchState> finalres = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iter = res.iterator();
SequenceSearchState curpat = iter.next();
finalres.add(curpat);
while (iter.hasNext()) {
SequenceSearchState nextpat = iter.next();
int comp = curpat.compareTo(nextpat);
if (comp == 0) { // Identical states
curpat.merge(nextpat);
}
else {
curpat = nextpat;
finalres.add(curpat);
}
}
return finalres;
}
/**
* Build a search state machine from a list of DittedBitSequences
* @param patterns bit sequence patterns
* @return search state the will match the given sequences
*/
static public SequenceSearchState buildStateMachine(
ArrayList<? extends DittedBitSequence> patterns) {
SequenceSearchState root = new SequenceSearchState(null);
for (int i = 0; i < patterns.size(); ++i) {
DittedBitSequence pat = patterns.get(i);
pat.setIndex(i);
root.addSequence(pat, 0);
}
root.sortSequences();
ArrayList<SequenceSearchState> statelevel = new ArrayList<SequenceSearchState>();
statelevel.add(root);
int level = 0;
do {
statelevel = buildTransitionLevel(statelevel, level);
level += 1;
}
while (!statelevel.isEmpty());
return root;
}
}

155
Ghidra/Features/BytePatterns/src/main/java/ghidra/app/analyzers/FunctionStartAnalyzer.java
View file

@ -15,8 +15,6 @@
*/
package ghidra.app.analyzers;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Iterator;
@ -51,7 +49,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
private static final String DESCRIPTION =
"Search for architecture specific byte patterns: typically starts of functions";
private static final String PRE_FUNCTION_MATCH_PROPERTY_NAME = "PreFunctionMatch";
private static final int RESTRICTED_PATTERN_BYTE_RANGE = 32;
private final static String OPTION_NAME_DATABLOCKS = "Search Data Blocks";
private static final String OPTION_DESCRIPTION_DATABLOCKS =
"Search for byte patterns in blocks that are not executable";
@ -214,7 +211,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
applyActionToSet(program, addr, funcResult, match);
}
protected boolean checkPreRequisites(Program program, Address addr) {
/**
* If the match's mark point occurs in undefined data, schedule disassembly
@ -258,7 +254,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
return true;
}
protected void applyActionToSet(Program program, Address addr, AddressSet resultSet,
Match match) {
@ -632,25 +627,34 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
return false;
}
AddressSet restrictedSet = removeNotSearchedAddresses(program, set);
boolean doExecutableBlocksOnly = checkForExecuteBlock(program) && executableBlocksOnly;
// clear out any previous potential matches, because we are re-looking at these places
// this will keep cruft from accumulating in the property map.
getOrCreatePotentialMatchPropertyMap(program).remove(restrictedSet);
getOrCreatePotentialMatchPropertyMap(program).remove(set);
MemoryBlock[] blocks = program.getMemory().getBlocks();
for (MemoryBlock block2 : blocks) {
MemoryBlock block = block2;
if (!restrictedSet.intersects(block.getStart(), block.getEnd())) {
continue;
MemoryBytePatternSearcher patternSearcher;
patternSearcher = new MemoryBytePatternSearcher("Function Starts", root) {
@Override
public void preMatchApply(MatchAction[] actions, Address addr) {
contextValueList = null; // make sure, only context from these actions used
}
try {
searchBlock(root, program, block, restrictedSet, monitor);
@Override
public void postMatchApply(MatchAction[] actions, Address addr) {
// Actions might have set context, check if postcondition failed first
if (!postreqFailedResult.contains(addr)) {
setCurrentContext(program, addr);
}
// get rid of the context list.
contextValueList = null;
}
catch (IOException e) {
log.appendMsg("Unable to scan block " + block.getName() + " for function starts");
}
}
};
patternSearcher.setSearchExecutableOnly(doExecutableBlocksOnly);
patternSearcher.search(program, set, monitor);
AutoAnalysisManager analysisManager = AutoAnalysisManager.getAnalysisManager(program);
if (!disassemResult.isEmpty()) {
analysisManager.disassemble(disassemResult);
@ -708,34 +712,17 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
}
/**
* Get rid of any blocks from the address set that shouldn't be searched.
*
* @param program
* @param bset
* @return
* @return true - if there are any blocks marked executable
*/
private AddressSet removeNotSearchedAddresses(Program program, AddressSetView bset) {
AddressSet restrictedSet = new AddressSet(bset);
private boolean checkForExecuteBlock(Program program) {
MemoryBlock[] blocks = program.getMemory().getBlocks();
boolean hasExecutable = false;
for (MemoryBlock block : blocks) {
if (block.isExecute()) {
hasExecutable = true;
return true;
}
}
for (MemoryBlock block : blocks) {
if (!block.isInitialized()) {
restrictedSet.deleteRange(block.getStart(), block.getEnd());
continue;
}
if (executableBlocksOnly && hasExecutable) {
if (!block.isExecute()) {
restrictedSet.deleteRange(block.getStart(), block.getEnd());
continue;
}
}
}
return restrictedSet;
return false;
}
@Override
@ -816,94 +803,6 @@ public class FunctionStartAnalyzer extends AbstractAnalyzer implements PatternFa
return patlist;
}
/**
* Search through bytes of a memory block using the finite state machine -root-
* Apply any additional rules for matching patterns.
* @param program is the Program being searched
* @param block is the specific block of bytes being searched
* @throws IOException
* @throws CancelledException
*/
private void searchBlock(SequenceSearchState root, Program program, MemoryBlock block,
AddressSetView restrictSet, TaskMonitor monitor)
throws IOException, CancelledException {
// if no restricted set, make restrict set the full block
AddressSet doneSet = new AddressSet(restrictSet);
if (doneSet.isEmpty()) {
doneSet.addRange(block.getStart(), block.getEnd());
}
doneSet = doneSet.intersectRange(block.getStart(), block.getEnd());
Address blockStartAddr = block.getStart();
// pull each range off the restricted set
AddressRangeIterator addressRanges = doneSet.getAddressRanges();
while (addressRanges.hasNext()) {
monitor.checkCanceled();
AddressRange addressRange = addressRanges.next();
monitor.setMessage("Function Search");
monitor.initialize(doneSet.getNumAddresses());
monitor.setProgress(0);
ArrayList<Match> mymatches = new ArrayList<>();
long streamoffset = blockStartAddr.getOffset();
// Give block a starting/ending point before this address to search
// patterns might start before, since they have a pre-pattern
// TODO: this is dangerous, since pattern might be very big, but the set should be restricted
// normally only when we are searching for more matching patterns that had a postrule that didn't satisfy
// normally the whole memory blocks will get searched.
long blockOffset = addressRange.getMinAddress().subtract(blockStartAddr);
blockOffset = blockOffset - RESTRICTED_PATTERN_BYTE_RANGE;
if (blockOffset <= 0) {
// don't go before the block start
blockOffset = 0;
}
// compute number of bytes in the range + 1, and don't search more than that.
long maxBlockSearchLength =
addressRange.getMaxAddress().subtract(blockStartAddr) - blockOffset + 1;
InputStream data = block.getData();
data.skip(blockOffset);
root.apply(data, maxBlockSearchLength, mymatches, monitor);
monitor.checkCanceled();
monitor.setMessage("Function Search (Examine Matches)");
monitor.initialize(mymatches.size());
monitor.setProgress(0);
// TODO: DANGER there is much offset<-->address calculation here
// should be OK, since they are all relative to the block.
for (int i = 0; i < mymatches.size(); ++i) {
monitor.checkCanceled();
monitor.setProgress(i);
Match match = mymatches.get(i);
Address addr = blockStartAddr.add(match.getMarkOffset() + blockOffset);
if (!match.checkPostRules(streamoffset + blockOffset)) {
continue;
}
MatchAction[] matchactions = match.getMatchActions();
contextValueList = null; // make sure, only context from these actions used
for (MatchAction matchaction : matchactions) {
matchaction.apply(program, addr, match);
}
// Actions might have set context, check if postcondition failed first
if (!postreqFailedResult.contains(addr)) {
setCurrentContext(program, addr);
}
else {
// didn't apply it, get rid of the context list.
contextValueList = null;
}
}
}
}
@Override
public MatchAction getMatchActionByName(String nm) {
if (nm.equals("funcstart")) {

55
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/AlignRule.java
View file

@ -1,55 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import ghidra.util.xml.SpecXmlUtils;
import ghidra.xml.XmlElement;
import ghidra.xml.XmlPullParser;
public class AlignRule implements PostRule {
private int mark; // Position, relative to start of pattern, to check alignment at
private int alignmask; // Mask of bits that must be zero
public AlignRule(){
}
public AlignRule(int mark, int alignmask){
this.mark = mark;
this.alignmask = alignmask;
}
@Override
public boolean apply(Pattern pat, long matchoffset) {
int off = (int)matchoffset;
return (((off + mark) & alignmask)==0);
}
@Override
public void restoreXml(XmlPullParser parser) {
XmlElement el = parser.start("align");
mark = SpecXmlUtils.decodeInt(el.getAttribute("mark"));
int bits = SpecXmlUtils.decodeInt(el.getAttribute("bits"));
alignmask = (1<<bits) - 1;
parser.end();
}
public int getAlignMask() {
return alignmask;
}
}

385
Ghidra/Features/BytePatterns/src/main/java/ghidra/util/bytesearch/SequenceSearchState.java
View file

@ -1,385 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.util.bytesearch;
import ghidra.util.task.TaskMonitor;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
public class SequenceSearchState implements Comparable<SequenceSearchState> {
private SequenceSearchState parent;
private ArrayList<DittedBitSequence> possible; // Patterns that could still match in this state
private ArrayList<DittedBitSequence> success; // Patterns that have matched successfully if we reached this state
private SequenceSearchState[] trans; // State transitions based on next byte
public SequenceSearchState(SequenceSearchState par) {
parent = par;
possible = new ArrayList<DittedBitSequence>();
success = null;
trans = null;
}
public int getMaxSequenceSize() {
int max = 0;
for(int i=0;i<possible.size();++i) {
int val = possible.get(i).getSize();
if (val > max)
max = val;
}
return max;
}
public void addSequence(DittedBitSequence pat,int pos) {
possible.add(pat);
if (pos == pat.getSize()) {
if (success == null)
success = new ArrayList<DittedBitSequence>();
success.add(pat);
}
}
public void sortSequences() {
Comparator<DittedBitSequence> comp = new Comparator<DittedBitSequence>() {
@Override
public int compare(DittedBitSequence o1, DittedBitSequence o2) {
return o1.getIndex() - o2.getIndex();
}
};
Collections.sort(possible,comp);
if (success != null)
Collections.sort(success,comp);
}
@Override
public int compareTo(SequenceSearchState o) {
int i=0;
for(;;) {
if (possible.size() <= i) {
if (o.possible.size() <=i)
return 0;
return -1;
}
if (o.possible.size() <= i)
return 1;
int indus = possible.get(i).getIndex(); // Lexicographic compare an sequence of sequences
int indthem = o.possible.get(i).getIndex();
if (indus != indthem)
return (indus < indthem) ? -1 : 1;
i += 1;
}
}
private void buildSingleTransition(ArrayList<SequenceSearchState> all,int pos,int val) {
SequenceSearchState newstate = null;
for(int i=0;i<possible.size();++i) {
DittedBitSequence curpat = possible.get(i);
if (curpat.isMatch(pos, val)) {
if (newstate == null)
newstate = new SequenceSearchState(this);
newstate.addSequence(curpat, pos+1);
}
}
trans[val] = newstate;
if (newstate != null) {
newstate.sortSequences();
all.add(newstate);
}
}
private void exportSuccess(ArrayList<Match> match,int offset) {
for(int i=0;i<success.size();++i) { // If we found matches
Match newmatch = new Match(success.get(i),offset);
match.add(newmatch);
}
}
/**
* Merge in -op- and this as a single state
* @param op
*/
private void merge(SequenceSearchState op) {
SequenceSearchState parent = op.parent;
for(int i=0;i<256;++i) {
if (parent.trans[i] == op) // Any references to -op- in parent
parent.trans[i] = this; // Should be replaced with this
}
if (op.success != null) { // Merge
if (success == null) {
success = op.success;
}
else {
ArrayList<DittedBitSequence> tmp = new ArrayList<DittedBitSequence>();
int i=0;
int j=0;
int curpat = -1;
int thispat = success.get(i).index;
int oppat = op.success.get(j).index;
while((i<success.size())||(j<op.success.size())) {
if (thispat == oppat) {
if (curpat != thispat) {
tmp.add(success.get(i));
curpat = thispat;
}
i += 1;
j += 1;
thispat = (i==success.size()) ? 10000000 : success.get(i).index;
oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
}
else if (thispat < oppat) {
if (curpat != thispat) {
tmp.add(success.get(i));
curpat = thispat;
}
i += 1;
thispat = (i==success.size()) ? 10000000 : success.get(i).index;
}
else {
if (curpat != oppat) {
tmp.add(op.success.get(j));
curpat = oppat;
}
j += 1;
oppat = (j==op.success.size()) ? 10000000 : op.success.get(j).index;
}
}
success = tmp;
}
}
}
public void sequenceMatch(byte[] bytearray,int numbytes,ArrayList<Match> match) {
int subindex = 0;
SequenceSearchState curstate = this;
do {
if (curstate.success != null)
curstate.exportSuccess(match, 0);
if (subindex >= numbytes) return;
curstate = curstate.trans[ 0xff & bytearray[subindex] ]; // Perform state transition based on next byte in buffer
subindex += 1;
} while(curstate != null);
}
/**
* Search for patterns in a byte array. All matches are returned.
* @param buffer is the array of bytes to search
* @param match is populated with a Match object for each pattern and position that matches
*/
public void apply(byte[] buffer,ArrayList<Match> match) {
SequenceSearchState curstate;
int subindex;
for(int offset=0;offset<buffer.length;++offset) {
curstate = this; // New starting offset -> Root state
subindex = offset;
do {
if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point
curstate.exportSuccess(match, offset);
if (subindex >= buffer.length) { // if we've run out of bytes, must restart at next offset
break;
}
curstate = curstate.trans[ 0xff & buffer[subindex] ]; // Perform state transition based on next byte
subindex += 1;
} while(curstate != null);
}
}
/**
* Search for pattern in the stream -in-.
* @param in - The stream to scan for matches
* @param match - Any matches are appended as Match records to this ArrayList
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in,ArrayList<Match> match,TaskMonitor monitor) throws IOException {
apply(in,-1L,match,monitor);
}
/**
* Search for pattern in the stream -in-.
* @param in - The stream to scan for matches
* @param maxBytes - The maximum number of bytes to scan forward in this stream
* @param match - Any matches are appended as Match records to this ArrayList
* @param monitor - if non-null, check for user cancel, and maintain progress info
* @throws IOException
*/
public void apply(InputStream in, long maxBytes, ArrayList<Match> match,TaskMonitor monitor) throws IOException {
int maxsize = getMaxSequenceSize()+1;
if (maxsize <4096)
maxsize = 4096;
if (maxBytes > 0) {
maxBytes += getMaxSequenceSize()+1;
}
byte[] firstbuf=new byte[maxsize];
byte[] secondbuf=new byte[maxsize];
byte[] curbuf;
SequenceSearchState curstate;
int fullbuffers; // Number of buffers that are completely full
int ra = in.read(firstbuf);
if (ra == firstbuf.length) {
ra = in.read(secondbuf);
if (ra == secondbuf.length) {
fullbuffers = 2;
}
else {
if (ra < 0)
ra = 0;
fullbuffers = 1;
byte[] tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = secondbuf[i];
secondbuf = tmp;
}
}
else if (ra < 0)
return; // No bytes at all were read
else {
byte[] tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = firstbuf[i];
firstbuf = tmp;
fullbuffers = 0;
secondbuf = new byte[0];
}
int offset=0;
int bufreloff=0;
int subindex;
while(fullbuffers == 2) {
curstate = this; // New starting offset -> Root state
subindex = bufreloff;
curbuf = firstbuf;
do {
if (curstate.success != null) // Check for any successful pattern matches for bytes up to this point
curstate.exportSuccess(match, offset);
if (subindex >= curbuf.length) { // check that we have enough bytes in current buffer
curbuf = secondbuf; // If not, switch to secondary buffer
subindex = 0;
}
curstate = curstate.trans[ 0xff & curbuf[subindex] ]; // Perform state transition based on next byte in buffer
subindex += 1;
} while(curstate != null);
offset += 1; // Advance to next starting offset
if (maxBytes > 0 && offset > maxBytes) {
break;
}
bufreloff += 1;
if (bufreloff == firstbuf.length) { // If starting offset no longer falls in firstbuf
byte[] tmp = firstbuf; // Switch firstbuf with secondbuf
firstbuf = secondbuf;
secondbuf = tmp;
ra = in.read(secondbuf); // refill secondbuf (old firstbuf) with new bytes
if (monitor!=null) {
if (monitor.isCancelled()) return;
monitor.setProgress(offset);
}
if (ra != secondbuf.length) {
fullbuffers = 1;
if (ra < 0)
ra = 0;
tmp = new byte[ra];
for(int i=0;i<ra;++i)
tmp[i] = secondbuf[i];
secondbuf = tmp;
}
bufreloff = 0;
}
}
while(fullbuffers >= 0 && (maxBytes <= 0 || offset < maxBytes)) {
if (secondbuf.length == 0)
fullbuffers = 0;
curstate = this;
subindex = bufreloff;
curbuf = firstbuf;
do {
if (curstate.success != null)
curstate.exportSuccess(match, offset);
if (subindex >= curbuf.length) {
if (curbuf == secondbuf) break; // Out of data, all pending patterns fail
curbuf = secondbuf;
subindex = 0;
if (curbuf.length==0) break;
}
curstate = curstate.trans[ 0xff & curbuf[subindex] ];
subindex += 1;
} while(curstate != null);
offset += 1;
bufreloff += 1;
if (bufreloff == firstbuf.length) {
if (fullbuffers == 0) break;
firstbuf = secondbuf;
fullbuffers = 0;
bufreloff = 0;
secondbuf = new byte[0];
}
}
}
static public ArrayList<SequenceSearchState> buildTransitionLevel(ArrayList<SequenceSearchState> prev,int pos) {
ArrayList<SequenceSearchState> res = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iterator = prev.iterator();
while(iterator.hasNext()) { // For each current state
SequenceSearchState next = iterator.next();
next.trans = new SequenceSearchState[256];
for(int i=0;i<256;++i) { // Try every byte transition
next.buildSingleTransition(res, pos, i);
}
}
if (res.isEmpty()) return res;
// Prepare to dedup the states
Collections.sort(res);
ArrayList<SequenceSearchState> finalres = new ArrayList<SequenceSearchState>();
Iterator<SequenceSearchState> iter = res.iterator();
SequenceSearchState curpat = iter.next();
finalres.add(curpat);
while(iter.hasNext()) {
SequenceSearchState nextpat = iter.next();
int comp = curpat.compareTo(nextpat);
if (comp == 0) { // Identical states
curpat.merge(nextpat);
}
else {
curpat = nextpat;
finalres.add(curpat);
}
}
return finalres;
}
static public SequenceSearchState buildStateMachine(ArrayList<? extends DittedBitSequence> patterns) {
SequenceSearchState root = new SequenceSearchState(null);
for(int i=0;i<patterns.size();++i) {
DittedBitSequence pat = patterns.get(i);
pat.index = i;
root.addSequence(pat, 0);
}
root.sortSequences();
ArrayList<SequenceSearchState> statelevel = new ArrayList<SequenceSearchState>();
statelevel.add(root);
int level = 0;
do {
statelevel = buildTransitionLevel(statelevel, level);
level += 1;
} while(!statelevel.isEmpty());
return root;
}
}

25
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/AbstractCreateDataBackgroundCmd.java
View file

@ -38,7 +38,7 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
extends BackgroundCommand {
protected final String name;
protected final Address address;
private Address address;
protected final int count;
protected final DataValidationOptions validationOptions;
protected final DataApplyOptions applyOptions;
@ -141,7 +141,8 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
* @return true if the data type creation completes successfully.
* @throws CancelledException if the user cancels this task.
*/
private boolean doApplyTo(Program program, TaskMonitor taskMonitor) throws CancelledException {
protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
throws CancelledException {
try {
monitor = taskMonitor;
@ -353,4 +354,24 @@ public abstract class AbstractCreateDataBackgroundCmd<T extends AbstractCreateDa
return (applyOptions.shouldClearDefinedData()) ? ClearDataMode.CLEAR_ALL_CONFLICT_DATA
: ClearDataMode.CLEAR_ALL_UNDEFINED_CONFLICT_DATA;
}
/**
* Get the address for the data item to be processed by the base implementation.
* In general this is the initial model address set when the command was created.
*
* @return the address of the data item being created.
*/
final protected Address getDataAddress() {
return address;
}
/**
* Set the address of the data item to be applied.
* Can be used for sub classes that need to apply multiple data items.
*
* @param addr set the current data address
*/
final protected void setDataAddress(Address addr) {
address = addr;
}
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/CreateTypeDescriptorBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateTypeDescriptorBackgroundCmd
private void loadModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new TypeDescriptorModel(program, address, validationOptions);
model = new TypeDescriptorModel(program, getDataAddress(), validationOptions);
}
}
@ -138,13 +138,13 @@ public class CreateTypeDescriptorBackgroundCmd
String prefix = demangledName + " ";
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, address,
EHDataTypeUtilities.createPlateCommentIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
applyOptions);
monitor.checkCanceled();
// Label
EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, address,
EHDataTypeUtilities.createSymbolIfNeeded(program, prefix, RTTI_0_NAME, null, getDataAddress(),
applyOptions);
return true;

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHCatchHandlerMapBackgroundCmd.java
View file

@ -74,7 +74,7 @@ public class CreateEHCatchHandlerMapBackgroundCmd
@Override
protected EHCatchHandlerModel createModel(Program program) {
if (model == null) {
model = new EHCatchHandlerModel(program, count, address, validationOptions);
model = new EHCatchHandlerModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHESTypeListBackgroundCmd.java
View file

@ -70,7 +70,7 @@ public class CreateEHESTypeListBackgroundCmd
@Override
protected EHESTypeListModel createModel(Program program) {
if (model == null) {
model = new EHESTypeListModel(program, address, validationOptions);
model = new EHESTypeListModel(program, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHFuncInfoBackgroundCmd.java
View file

@ -71,7 +71,7 @@ public class CreateEHFuncInfoBackgroundCmd
@Override
protected EHFunctionInfoModel createModel(Program program) {
if (model == null) {
model = new EHFunctionInfoModel(program, address, validationOptions);
model = new EHFunctionInfoModel(program, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHIPToStateMapBackgroundCmd.java
View file

@ -72,7 +72,7 @@ public class CreateEHIPToStateMapBackgroundCmd
@Override
protected EHIPToStateModel createModel(Program program) {
if (model == null) {
model = new EHIPToStateModel(program, count, address, validationOptions);
model = new EHIPToStateModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHTryBlockMapBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateEHTryBlockMapBackgroundCmd
@Override
protected EHTryBlockModel createModel(Program program) {
if (model == null) {
model = new EHTryBlockModel(program, count, address, validationOptions);
model = new EHTryBlockModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

2
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/exceptionhandling/CreateEHUnwindMapBackgroundCmd.java
View file

@ -73,7 +73,7 @@ public class CreateEHUnwindMapBackgroundCmd extends AbstractCreateDataBackground
@Override
protected EHUnwindModel createModel(Program program) {
if (model == null) {
model = new EHUnwindModel(program, count, address, validationOptions);
model = new EHUnwindModel(program, count, getDataAddress(), validationOptions);
}
return model;
}

8
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti1BackgroundCmd.java
View file

@ -63,7 +63,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti1Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti1Model(program, address, validationOptions);
model = new Rtti1Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -101,14 +101,14 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
}
catch (InvalidDataTypeException e) {
// Couldn't get pmd and attributes so leave it off and simply log the error.
String message = "Unable to get PMD and attributes for RTTI1 at " + address + ".";
String message = "Unable to get PMD and attributes for RTTI1 at " + getDataAddress() + ".";
handleError(message);
}
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_1_NAME, suffix, address, applyOptions);
RTTI_1_NAME, suffix, getDataAddress(), applyOptions);
monitor.checkCanceled();
@ -116,7 +116,7 @@ public class CreateRtti1BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
if (applyOptions.shouldCreateLabel()) {
String rtti1Suffix = RTTI_1_NAME + suffix;
rtti1Suffix = SymbolUtilities.replaceInvalidChars(rtti1Suffix, true);
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, rtti1Suffix);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, rtti1Suffix);
}
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti2BackgroundCmd.java
View file

@ -68,7 +68,7 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti2Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti2Model(program, rtti1Count, address, validationOptions);
model = new Rtti2Model(program, rtti1Count, getDataAddress(), validationOptions);
}
return model;
}
@ -118,13 +118,13 @@ public class CreateRtti2BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_2_NAME, null, address, applyOptions);
RTTI_2_NAME, null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_2_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_2_NAME);
}
return true;
}

6
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti3BackgroundCmd.java
View file

@ -63,7 +63,7 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
@Override
protected Rtti3Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti3Model(program, address, validationOptions);
model = new Rtti3Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -105,13 +105,13 @@ public class CreateRtti3BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_3_NAME, null, address, applyOptions);
RTTI_3_NAME, null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_3_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model, RTTI_3_NAME);
}
}

177
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateRtti4BackgroundCmd.java
View file

@ -15,29 +15,32 @@
*/
package ghidra.app.cmd.data.rtti;
import java.util.List;
import java.util.Set;
import java.util.*;
import ghidra.app.cmd.data.*;
import ghidra.app.util.datatype.microsoft.DataApplyOptions;
import ghidra.app.util.datatype.microsoft.DataValidationOptions;
import ghidra.program.model.address.Address;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.data.InvalidDataTypeException;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.model.symbol.Namespace;
import ghidra.program.util.ProgramMemoryUtil;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
/**
* This command will create an RTTI4 data type.
* If there are any existing instructions in the area to be made into data, the command will fail.
* This command will create multiple RTTI4 data types all at one time.
* If there are any existing instructions in the areas to be made into data, the command will fail only on that RTTI4 entry.
* Any data in the area will be replaced with the new dataType.
*/
public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rtti4Model> {
private static final String RTTI_4_NAME = "RTTI Complete Object Locator";
private List<MemoryBlock> vfTableBlocks;
private List<Address> rtti4Locations;
/**
* Constructs a command for applying an RTTI4 dataType at an address.
@ -54,12 +57,44 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
super(Rtti4Model.DATA_TYPE_NAME, address, 1, validationOptions, applyOptions);
this.vfTableBlocks = vfTableBlocks;
rtti4Locations = new ArrayList<Address>();
rtti4Locations.add(address);
}
public CreateRtti4BackgroundCmd(List<Address> addresses, List<MemoryBlock> vfTableBlocks,
DataValidationOptions validationOptions, DataApplyOptions applyOptions) {
super(Rtti4Model.DATA_TYPE_NAME, addresses.get(0), 1, validationOptions, applyOptions);
this.rtti4Locations = addresses;
this.vfTableBlocks = vfTableBlocks;
}
@Override
protected boolean doApplyTo(Program program, TaskMonitor taskMonitor)
throws CancelledException {
// process each potential RTTI4 entry, keeping track of good RTTI4 tables
List<Address> goodRtti4Locations = new ArrayList<Address>();
boolean succeeded = false;
for (Address addr : rtti4Locations) {
setDataAddress(addr);
succeeded |= super.doApplyTo(program, taskMonitor);
goodRtti4Locations.add(addr);
}
// if any succeeded and should create associated data, make the vftables all at one time
if (succeeded && applyOptions.shouldFollowData()) {
createAssociatedVfTables(program, goodRtti4Locations, taskMonitor);
}
return succeeded;
}
@Override
protected Rtti4Model createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new Rtti4Model(program, address, validationOptions);
if (model == null || program != model.getProgram() ||
!getDataAddress().equals(model.getAddress())) {
model = new Rtti4Model(program, getDataAddress(), validationOptions);
}
return model;
}
@ -75,7 +110,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
}
catch (InvalidDataTypeException e) {
createRtti0Success = false;
// log message and continue with other markup.
// log message and continue with other mark-up.
handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
}
@ -89,9 +124,7 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
handleErrorMessage(model.getProgram(), model.getAddress(), e.getMessage());
}
boolean createVfTableSuccess = createVfTable();
return createRtti0Success && createRtti3Success && createVfTableSuccess;
return createRtti0Success && createRtti3Success;
}
private boolean createRtti0() throws CancelledException, InvalidDataTypeException {
@ -112,57 +145,95 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
return cmd.applyTo(model.getProgram(), monitor);
}
private boolean createVfTable() throws CancelledException {
private boolean createAssociatedVfTables(Program program, List<Address> goodRtti4Locations,
TaskMonitor taskMonitor) throws CancelledException {
monitor.checkCanceled();
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI4 Vftables");
Program program = model.getProgram();
HashMap<Address, VfTableModel> foundVFtables = new HashMap<>();
Address rtti4Address = address;
int defaultPointerSize = program.getDefaultPointerSize();
int alignment = defaultPointerSize; // Align the vf table based on the size of the pointers in it.
Set<Address> directRtti4Refs =
ProgramMemoryUtil.findDirectReferences(program, vfTableBlocks, alignment, rtti4Address,
monitor);
for (Address rtti4Address : goodRtti4Locations) {
VfTableModel validVfTableModel = null;
for (Address possibleVfMetaAddr : directRtti4Refs) {
byte[] bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti4Address);
addByteSearchPattern(searcher, foundVFtables, rtti4Address, bytes);
}
AddressSet searchSet = new AddressSet();
for (MemoryBlock block : vfTableBlocks) {
searchSet.add(block.getStart(), block.getEnd());
}
searcher.search(program, searchSet, monitor);
// did the search, now process the results
boolean didSome = false;
for (Address rtti4Address : goodRtti4Locations) {
monitor.checkCanceled();
Address possibleVfTableAddr = possibleVfMetaAddr.add(defaultPointerSize);
VfTableModel vfTableModel = foundVFtables.get(rtti4Address);
if (vfTableModel == null) {
String message =
"No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(program, rtti4Address, message);
continue;
}
// Validate the model. Don't apply the command if invalid.
try {
VfTableModel vfTableModel =
new VfTableModel(program, possibleVfTableAddr, validationOptions);
vfTableModel.validate();
CreateVfTableBackgroundCmd cmd =
new CreateVfTableBackgroundCmd(vfTableModel, applyOptions);
didSome |= cmd.applyTo(program, monitor);
}
if (validVfTableModel != null) {
String message = "More than one possible vfTable found for " +
Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(program, rtti4Address, message);
return false;
return didSome;
}
/**
* Add a search pattern, to the searcher, for the set of bytes representing an rtti4 location.
* Only one VFTable for is allowed for an RTT4 location, last one in wins and gets created.
*
* @param searcher byte pattern searcher
* @param foundVFtables list of addresses accumulated when actual search is performed
* @param rtti4Address location of rttiAddress to find vfTable for
* @param bytes bytes representing rtti4Addres to be found in memory
*/
private void addByteSearchPattern(MemoryBytePatternSearcher searcher,
HashMap<Address, VfTableModel> foundVFtables, Address rtti4Address, byte[] bytes) {
if (bytes == null) {
return;
}
GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti4Address) {
@Override
public void apply(Program prog, Address addr, Match match) {
Address possibleVfTableAddr = addr.add(prog.getDefaultPointerSize());
// See if VfTable is valid, and add to rtti4 to vfTable map
try {
VfTableModel vfTableModel =
new VfTableModel(prog, possibleVfTableAddr, validationOptions);
vfTableModel.validate();
VfTableModel existing = foundVFtables.put(rtti4Address, vfTableModel);
if (existing != null) {
// potential table already found, is an error, don't know which is right
String message = "More than one possible vfTable found for " +
Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(prog, rtti4Address, message);
}
}
catch (InvalidDataTypeException e) {
// This isn't a valid model.
}
validVfTableModel = vfTableModel;
}
catch (InvalidDataTypeException e) {
continue; // This isn't a valid model.
}
}
};
if (validVfTableModel == null) {
String message =
"No vfTable found for " + Rtti4Model.DATA_TYPE_NAME + " @ " + rtti4Address;
handleErrorMessage(program, rtti4Address, message);
return false;
}
GenericByteSequencePattern<Address> genericByteMatchPattern =
new GenericByteSequencePattern<Address>(bytes, action);
monitor.checkCanceled();
CreateVfTableBackgroundCmd cmd =
new CreateVfTableBackgroundCmd(validVfTableModel, applyOptions);
return cmd.applyTo(program, monitor);
searcher.addPattern(genericByteMatchPattern);
}
@Override
@ -177,20 +248,20 @@ public class CreateRtti4BackgroundCmd extends AbstractCreateDataBackgroundCmd<Rt
if (rtti0Model != null) {
// Plate Comment
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
RTTI_4_NAME, null, address, applyOptions);
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER, RTTI_4_NAME,
null, getDataAddress(), applyOptions);
monitor.checkCanceled();
// Label
if (applyOptions.shouldCreateLabel()) {
RttiUtil.createSymbolFromDemangledType(program, address, rtti0Model, RTTI_4_NAME);
RttiUtil.createSymbolFromDemangledType(program, getDataAddress(), rtti0Model,
RTTI_4_NAME);
}
}
return true;
}
}

19
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/CreateVfTableBackgroundCmd.java
View file

@ -15,7 +15,7 @@
*/
package ghidra.app.cmd.data.rtti;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.getAbsoluteAddress;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.*;
import ghidra.app.cmd.data.*;
import ghidra.app.util.datatype.microsoft.DataApplyOptions;
@ -70,7 +70,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
@Override
protected VfTableModel createModel(Program program) {
if (model == null || program != model.getProgram()) {
model = new VfTableModel(program, address, validationOptions);
model = new VfTableModel(program, getDataAddress(), validationOptions);
}
return model;
}
@ -105,7 +105,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
return false;
}
long displacement = dataType.getLength();
Address terminatorAddress = address.add(displacement);
Address terminatorAddress = getDataAddress().add(displacement);
try {
Address referencedAddress = getAbsoluteAddress(program, terminatorAddress);
if (referencedAddress == null || referencedAddress.getOffset() != 0) {
@ -136,7 +136,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
private boolean createMetaPointer() {
Program program = model.getProgram();
Address metaAddress = address.subtract(program.getDefaultPointerSize());
Address metaAddress = getDataAddress().subtract(program.getDefaultPointerSize());
// Create a pointer to the RTTI4 associated with the vf table.
DataType metaPointer = new PointerDataType(program.getDataTypeManager());
@ -162,7 +162,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
private boolean createVfTableMarkup() throws CancelledException, InvalidDataTypeException {
Address vfTableAddress = address;
Address vfTableAddress = getDataAddress();
Program program = model.getProgram();
monitor.checkCanceled();
@ -172,9 +172,8 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
if (rtti0Model != null) {
// Plate Comment
EHDataTypeUtilities.createPlateCommentIfNeeded(program,
RttiUtil.CONST_PREFIX + RttiUtil.getDescriptorTypeNamespace(rtti0Model) +
Namespace.DELIMITER,
EHDataTypeUtilities.createPlateCommentIfNeeded(program, RttiUtil.CONST_PREFIX +
RttiUtil.getDescriptorTypeNamespace(rtti0Model) + Namespace.DELIMITER,
VF_TABLE_LABEL, null, vfTableAddress, applyOptions);
monitor.checkCanceled();
@ -188,7 +187,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
// Create functions that are referred to by the vf table.
if (applyOptions.shouldCreateFunction()) {
int elementCount = model.getElementCount();
int elementCount = model.getCount();
for (int tableElementIndex = 0; tableElementIndex < elementCount; tableElementIndex++) {
monitor.checkCanceled();
Address vfPointer = model.getVirtualFunctionPointer(tableElementIndex);
@ -238,7 +237,7 @@ public class CreateVfTableBackgroundCmd extends AbstractCreateDataBackgroundCmd<
* the vftable.
*/
private Address getMetaAddress(Program program) {
return address.subtract(program.getDefaultPointerSize());
return getDataAddress().subtract(program.getDefaultPointerSize());
}
}

18
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/cmd/data/rtti/VfTableModel.java
View file

@ -15,7 +15,7 @@
*/
package ghidra.app.cmd.data.rtti;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.getAbsoluteAddress;
import static ghidra.app.util.datatype.microsoft.MSDataTypeUtils.*;
import ghidra.app.cmd.data.AbstractCreateDataTypeModel;
import ghidra.app.cmd.data.TypeDescriptorModel;
@ -40,7 +40,6 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
private DataType dataType;
private Rtti4Model rtti4Model;
private int elementCount = -1;
private Program lastProgram;
private DataType lastDataType;
@ -80,7 +79,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
long entrySize = individualEntryDataType.getLength();
// Each entry is a pointer to where a function can possibly be created.
long numEntries = RttiUtil.getVfTableCount(program, startAddress);
long numEntries = getCount();
if (numEntries == 0) {
throw new InvalidDataTypeException(
getName() + " data type at " + getAddress() + " doesn't have a valid vf table.");
@ -123,7 +122,7 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
lastDataType = null;
lastElementCount = -1;
lastElementCount = RttiUtil.getVfTableCount(program, getAddress());
lastElementCount = getCount();
if (lastElementCount > 0) {
DataTypeManager dataTypeManager = program.getDataTypeManager();
PointerDataType pointerDt = new PointerDataType(dataTypeManager);
@ -168,17 +167,6 @@ public class VfTableModel extends AbstractCreateDataTypeModel {
return getAbsoluteAddress(getProgram(), address);
}
/**
* Gets the number of elements in the vf table. Returns 0 if this model isn't for a valid vf table.
* @return the number of vf table elements or 0.
*/
public int getElementCount() {
if (elementCount == -1) {
elementCount = RttiUtil.getVfTableCount(getProgram(), getAddress());
}
return elementCount;
}
/**
* Gets the type descriptor (RTTI 0) model associated with this vf table.
* @return the type descriptor (RTTI 0) model or null.

191
Ghidra/Features/MicrosoftCodeAnalyzer/src/main/java/ghidra/app/plugin/prototype/MicrosoftCodeAnalyzerPlugin/RttiAnalyzer.java
View file

@ -30,6 +30,7 @@ import ghidra.program.model.lang.UndefinedValueException;
import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.util.ProgramMemoryUtil;
import ghidra.util.bytesearch.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor;
@ -41,7 +42,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
private static final String NAME = "Windows x86 PE RTTI Analyzer";
private static final String DESCRIPTION =
"This analyzer finds and creates all of the RTTI metadata structures and their associated vf tables.";
"Finds and creates RTTI metadata structures and associated vf tables.";
// TODO If we want the RTTI analyzer to find all type descriptors regardless of whether
// they are used for RTTI, then change the CLASS_PREFIX_CHARS to ".". Need to be
@ -133,6 +134,7 @@ public class RttiAnalyzer extends AbstractAnalyzer {
monitor.setMaximum(possibleRtti0Addresses.size());
monitor.setMessage("Creating RTTI Data...");
ArrayList<Address> rtti0Locations = new ArrayList<Address>();
int count = 0;
for (Address rtti0Address : possibleRtti0Addresses) {
monitor.checkCanceled();
@ -157,29 +159,29 @@ public class RttiAnalyzer extends AbstractAnalyzer {
rtti0Address, validationOptions, applyOptions);
typeDescCmd.applyTo(program, monitor);
// Create any valid RTTI4s for this TypeDescriptor
processRtti4sForRtti0(program, rtti0Address, monitor);
rtti0Locations.add(rtti0Address);
}
// Create any valid RTTI4s for this TypeDescriptor
processRtti4sForRtti0(program, rtti0Locations, monitor);
}
private void processRtti4sForRtti0(Program program, Address rtti0Address, TaskMonitor monitor)
throws CancelledException {
private void processRtti4sForRtti0(Program program, List<Address> rtti0Locations,
TaskMonitor monitor) throws CancelledException {
List<MemoryBlock> rDataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
List<MemoryBlock> dataBlocks = ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
program.getMemory(), ".rdata", monitor);
dataBlocks.addAll(ProgramMemoryUtil.getMemoryBlocksStartingWithName(program,
program.getMemory(), ".data", monitor));
List<Address> rtti4Addresses =
getRtti4Addresses(program, rDataBlocks, rtti0Address, validationOptions, monitor);
getRtti4Addresses(program, dataBlocks, rtti0Locations, validationOptions, monitor);
for (Address rtti4Address : rtti4Addresses) {
monitor.checkCanceled();
CreateRtti4BackgroundCmd cmd =
new CreateRtti4BackgroundCmd(rtti4Address, rDataBlocks, validationOptions,
applyOptions);
cmd.applyTo(program, monitor);
}
// create all found RTTI4 tables at once
CreateRtti4BackgroundCmd cmd = new CreateRtti4BackgroundCmd(rtti4Addresses, dataBlocks,
validationOptions, applyOptions);
cmd.applyTo(program, monitor);
}
/**
@ -187,67 +189,128 @@ public class RttiAnalyzer extends AbstractAnalyzer {
* the RTTI 0 at the indicated base address.
* @param program the program containing the RTTI 0 data structure.
* @param rtti4Blocks the memory blocks to be searched for RTTI4 structures.
* @param rtti0Address the base address of the RTTI 0 structure in the program
* @param rtti0Locations the base addresses of the RTTI 0 structure in the program
* @param validationOptions options indicating how validation is performed for data structures
* @param monitor the task monitor for cancelling a task
* @param monitor the task monitor for canceling a task
* @return the RTTI 4 base addresses associated with the RTTI 0
* @throws CancelledException if the user cancels this task.
*/
private static List<Address> getRtti4Addresses(Program program, List<MemoryBlock> rtti4Blocks,
Address rtti0Address, DataValidationOptions validationOptions, TaskMonitor monitor)
throws CancelledException {
List<Address> rtti0Locations, DataValidationOptions validationOptions,
TaskMonitor monitor) throws CancelledException {
monitor.checkCanceled();
List<Address> addresses = new ArrayList<>(); // the RTTI 4 addresses
int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset();
Set<Address> refsToRtti0 = getRefsToRtti0(program, rtti4Blocks, rtti0Address);
List<Address> addresses =
getRefsToRtti0(program, rtti4Blocks, rtti0Locations, validationOptions, monitor);
// for each RTTI 0 now see if we can get RTTI4s that refer to it.
for (Address refAddress : refsToRtti0) {
monitor.checkCanceled();
Address possibleRtti4Address;
try {
possibleRtti4Address = refAddress.subtractNoWrap(rtti0PointerOffset);
}
catch (AddressOverflowException e) {
continue; // Couldn't get an Rtti4 address.
}
Rtti4Model rtti4Model =
new Rtti4Model(program, possibleRtti4Address, validationOptions);
try {
rtti4Model.validate();
}
catch (InvalidDataTypeException e) {
continue; // Only process valid RTTI 4 data.
}
// Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly
// through the RTTI 3.
boolean refersToRtti0 = rtti4Model.refersToRtti0(rtti0Address);
if (!refersToRtti0) {
continue; // Only process valid RTTI 4 data.
}
addresses.add(possibleRtti4Address);
}
return addresses;
}
private static Set<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
Address rtti0Address) throws CancelledException {
Set<Address> refsToRtti0;
if (MSDataTypeUtils.is64Bit(program)) {
refsToRtti0 = ProgramMemoryUtil.findImageBaseOffsets32(program, 4, rtti0Address,
TaskMonitor.DUMMY);
/** For each of the RTTI0 locations found locate the associated RTTI4 structure referring to it.
*
* @param program program to be searched
* @param dataBlocks dataBlocks to search
* @param rtti0Locations list of known rtti0 locations
* @param validationOptions options for validation of found RTTI4 entries
* @param monitor to cancel
* @return list of found RTTI4 references to known RTTI0 locations
* @throws CancelledException if canceled
*/
private static List<Address> getRefsToRtti0(Program program, List<MemoryBlock> dataBlocks,
List<Address> rtti0Locations, DataValidationOptions validationOptions,
TaskMonitor monitor) throws CancelledException {
List<Address> addresses = new ArrayList<>(); // the RTTI 4 addresses
int rtti0PointerOffset = Rtti4Model.getRtti0PointerComponentOffset();
MemoryBytePatternSearcher searcher = new MemoryBytePatternSearcher("RTTI0 refernces");
for (Address rtti0Address : rtti0Locations) {
byte[] bytes;
if (MSDataTypeUtils.is64Bit(program)) {
// 64-bit programs will have the addresses as offsets from the image base (BOS)
bytes = ProgramMemoryUtil.getImageBaseOffsets32Bytes(program, 4, rtti0Address);
addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
rtti0Address, bytes);
}
else {
// 32-bit could have direct address in memory
bytes = ProgramMemoryUtil.getDirectAddressBytes(program, rtti0Address);
addByteSearchPattern(searcher, validationOptions, addresses, rtti0PointerOffset,
rtti0Address, bytes);
}
}
else {
refsToRtti0 = ProgramMemoryUtil.findDirectReferences(program, dataBlocks,
program.getDefaultPointerSize(), rtti0Address, TaskMonitor.DUMMY);
AddressSet searchSet = new AddressSet();
for (MemoryBlock block : dataBlocks) {
searchSet.add(block.getStart(), block.getEnd());
}
return refsToRtti0;
searcher.search(program, searchSet, monitor);
return addresses;
}
/**
* Add a search pattern, to the searcher, for the set of bytes representing an address
*
* @param searcher pattern searcher
* @param validationOptions RTTI4 validation options
* @param addresses list of found valid RTTI4 locations accumulated during actual search
* @param rtti0PointerOffset offset of pointer in RTTI4 entry to RTTI0
* @param rtti0Address RTTI0 address corresponding to pattern of bytes
* @param bytes pattern of bytes in memory corresponding to address
*/
private static void addByteSearchPattern(MemoryBytePatternSearcher searcher,
DataValidationOptions validationOptions, List<Address> addresses,
int rtti0PointerOffset, Address rtti0Address, byte[] bytes) {
// no pattern bytes.
if (bytes == null) {
return;
}
// Each time a match for this byte pattern validate as an RTTI4 and add to list
GenericMatchAction<Address> action = new GenericMatchAction<Address>(rtti0Address) {
@Override
public void apply(Program prog, Address addr, Match match) {
Address possibleRtti4Address;
try {
possibleRtti4Address = addr.subtractNoWrap(rtti0PointerOffset);
}
catch (AddressOverflowException e) {
return; // Couldn't get an Rtti4 address.
}
Rtti4Model rtti4Model =
new Rtti4Model(prog, possibleRtti4Address, validationOptions);
try {
rtti4Model.validate();
}
catch (InvalidDataTypeException e) {
return; // Only process valid RTTI 4 data.
}
// Check that the RTTI 0 is referred to both directly from the RTTI 4 and indirectly
// through the RTTI 3.
boolean refersToRtti0 = rtti4Model.refersToRtti0(getMatchValue());
if (!refersToRtti0) {
return; // Only process valid RTTI 4 data.
}
// add to list of RTTI4 locations to be processed later
addresses.add(possibleRtti4Address);
}
};
// create a Pattern of the bytes and the MatchAction to perform upon a match
GenericByteSequencePattern<Address> genericByteMatchPattern =
new GenericByteSequencePattern<Address>(bytes, action);
searcher.addPattern(genericByteMatchPattern);
}
}

25
Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/util/AddressSetPropertyMapDB.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +15,10 @@
*/
package ghidra.program.database.util;
import java.util.ConcurrentModificationException;
import db.*;
import db.util.ErrorHandler;
import ghidra.program.database.ProgramDB;
import ghidra.program.database.map.AddressMap;
import ghidra.program.model.address.*;
@ -26,11 +29,6 @@ import ghidra.util.exception.CancelledException;
import ghidra.util.exception.DuplicateNameException;
import ghidra.util.task.TaskMonitor;
import java.util.ConcurrentModificationException;
import db.*;
import db.util.ErrorHandler;
/**
* AddressSetPropertyMap that uses a RangeMapDB to maintain a set of addresses.
*
@ -74,8 +72,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
DBHandle dbh = program.getDBHandle();
String tableName = AddressSetPropertyMapDB.TABLE_PREFIX + mapName;
if (dbh.getTable(tableName) != null) {
throw new DuplicateNameException("Address Set Property Map named " + mapName +
" already exists.");
throw new DuplicateNameException(
"Address Set Property Map named " + mapName + " already exists.");
}
return new AddressSetPropertyMapDB(program, mapName, program, addrMap, lock);
@ -91,9 +89,8 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
this.mapName = mapName;
this.lock = lock;
propertyMap =
new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
propertyMap = new AddressRangeMapDB(program.getDBHandle(), program.getAddressMap(),
program.getLock(), MY_PREFIX + mapName, errHandler, BooleanField.class, true);
}
@Override
@ -114,7 +111,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
* @see ghidra.program.model.util.AddressSetPropertyMap#add(ghidra.program.model.address.AddressSet)
*/
@Override
public void add(AddressSet addressSet) {
public void add(AddressSetView addressSet) {
checkDeleted();
lock.acquire();
@ -135,7 +132,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
* @see ghidra.program.model.util.AddressSetPropertyMap#set(ghidra.program.model.address.AddressSet)
*/
@Override
public void set(AddressSet addressSet) {
public void set(AddressSetView addressSet) {
checkDeleted();
lock.acquire();
try {
@ -169,7 +166,7 @@ public class AddressSetPropertyMapDB implements AddressSetPropertyMap {
* @see ghidra.program.model.util.AddressSetPropertyMap#remove(ghidra.program.model.address.AddressSet)
*/
@Override
public void remove(AddressSet addressSet) {
public void remove(AddressSetView addressSet) {
checkDeleted();
lock.acquire();
try {

8
Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/util/AddressSetPropertyMap.java
View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -35,13 +34,14 @@ public interface AddressSetPropertyMap {
* Add the address set to the property map.
* @param addressSet address set to add
*/
void add(AddressSet addressSet);
void add(AddressSetView addressSet);
/**
* Clear the property map and set it with the given address set.
* @param addressSet address set to use
*/
void set(AddressSet addressSet);
void set(AddressSetView addressSet);
/**
* Remove the address range from the property map.
* @param start start of the range
@ -53,7 +53,7 @@ public interface AddressSetPropertyMap {
* Remove the address set from the property map.
* @param addressSet address set to remove
*/
void remove(AddressSet addressSet);
void remove(AddressSetView addressSet);
/**
* Return the address set for the property map.