Merge remote-tracking branch 'origin/caheckman_FillOutStructure'

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/MetaDataType.java

@@ -0,0 +1,110 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.program.model.data;
+
+public enum MetaDataType {
+	// Enumerations are ordered in terms of how "specific" the data-type class is
+	VOID,		// "void" data-type
+	UNKNOWN,	// An unknown/undefined data-type
+	INT,		// Signed integer
+	UINT,		// Unsigned integer
+	BOOL,		// Boolean
+	CODE,		// Executable code
+	FLOAT,		// Floating-point
+	PTR,		// Pointer
+	ARRAY,		// Array
+	STRUCT;		// Structured data-type
+
+	public static MetaDataType getMeta(DataType dt) {
+		if (dt instanceof TypeDef) {
+			dt = ((TypeDef) dt).getBaseDataType();
+		}
+		if (dt instanceof DefaultDataType || dt instanceof Undefined) {
+			return UNKNOWN;
+		}
+		if (dt instanceof AbstractIntegerDataType) {
+			if (dt instanceof BooleanDataType) {
+				return BOOL;
+			}
+			if (((AbstractIntegerDataType) dt).isSigned()) {
+				return INT;
+			}
+			return UINT;
+		}
+		if (dt instanceof Pointer) {
+			return PTR;
+		}
+		if (dt instanceof Array) {
+			return ARRAY;
+		}
+		if (dt instanceof Structure) {
+			return STRUCT;
+		}
+		if (dt instanceof AbstractFloatDataType) {
+			return FLOAT;
+		}
+		if (dt instanceof ArrayStringable) {
+			return INT;
+		}
+		if (dt instanceof FunctionDefinition) {
+			return CODE;
+		}
+		if (dt instanceof Enum) {
+			return UINT;
+		}
+		if (dt instanceof AbstractStringDataType) {
+			return ARRAY;
+		}
+		return STRUCT;
+	}
+
+	public static DataType getMostSpecificDataType(DataType a, DataType b) {
+		if (a == null) {
+			return b;
+		}
+		if (b == null) {
+			return a;
+		}
+		DataType aCopy = a;
+		DataType bCopy = b;
+		for (;;) {
+			MetaDataType aMeta = MetaDataType.getMeta(a);
+			MetaDataType bMeta = MetaDataType.getMeta(b);
+			int compare = aMeta.compareTo(bMeta);
+			if (compare < 0) {
+				return bCopy;
+			}
+			else if (compare > 0) {
+				return aCopy;
+			}
+			if (aMeta == MetaDataType.PTR) {
+				a = ((Pointer) a).getDataType();
+				b = ((Pointer) b).getDataType();
+			}
+			else if (aMeta == MetaDataType.ARRAY) {
+				if (!(a instanceof Array) || !(b instanceof Array)) {
+					break;
+				}
+				a = ((Array) a).getDataType();
+				b = ((Array) b).getDataType();
+			}
+			else {
+				break;
+			}
+		}
+		return aCopy;
+	}
+}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/data/NoisyStructureBuilder.java

@@ -0,0 +1,172 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.program.model.data;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+/**
+ * Build a structure from a "noisy" source of field information.
+ * Feed it field records, either via addDataType(), when we
+ * have more definitive info about the size of the field, or via addReference()
+ * when we have a pointer reference to the field with possibly less info about the field size.
+ * 
+ * As records come in, overlaps and conflicts in specific field data-types are resolved.
+ * In a conflict, less specific data-types are replaced.
+ * After all information is collected a final Structure can be built by iterating over
+ * the final field entries.
+ */
+public class NoisyStructureBuilder {
+	private TreeMap<Long, DataType> offsetToDataTypeMap = new TreeMap<Long, DataType>();
+	private Structure structDT = null;
+	private long sizeOfStruct = 0;
+
+	private void computeMax(long newOff, int length) {
+		if (sizeOfStruct < (newOff + length)) {
+			sizeOfStruct = newOff + length;
+		}
+	}
+
+	/**
+	 * Check if the given range overlaps any existing field entries.  If it does
+	 * return the first entry, otherwise return null.
+	 * @param offset is the starting of the range (in bytes)
+	 * @param size is the number of bytes in the range
+	 * @return the first overlapping entry or null
+	 */
+	private Entry<Long, DataType> checkForOverlap(long offset, int size) {
+		Entry<Long, DataType> res = offsetToDataTypeMap.floorEntry(offset);
+		if (res != null) {
+			long last = res.getKey().longValue() + res.getValue().getLength();
+			if (offset < last) {
+				return res;
+			}
+		}
+		res = offsetToDataTypeMap.higherEntry(offset);
+		if (res != null) {
+			long last = offset + size;
+			if (res.getKey() < last) {
+				return res;
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * @return the size of the structure in bytes (given current information)
+	 */
+	public long getSize() {
+		return sizeOfStruct;
+	}
+
+	/**
+	 * Add data-type information about a specific field
+	 * @param offset of the field within the structure
+	 * @param dt is the data-type of field if known (null otherwise)
+	 */
+	public void addDataType(long offset, DataType dt) {
+		if (dt == null) {
+			computeMax(offset, 1);
+			return;
+		}
+		computeMax(offset, dt.getLength());
+		Entry<Long, DataType> firstEntry = checkForOverlap(offset, dt.getLength());
+		if (firstEntry != null) {
+			if (firstEntry.getKey().longValue() == offset &&
+				firstEntry.getValue().getLength() == dt.getLength()) {
+				// Matching field,  compare the data-types
+				if (dt != MetaDataType.getMostSpecificDataType(firstEntry.getValue(), dt)) {
+					return;
+				}
+			}
+			else if (firstEntry.getKey().longValue() <= offset &&
+				offset + dt.getLength() < firstEntry.getKey().longValue() +
+					firstEntry.getValue().getLength()) {
+				// Completely contained within preexisting entry
+				if (!(firstEntry.getValue() instanceof Undefined)) {
+					// Don't override preexisting entry with a smaller one
+					return;
+				}
+				// unless the preexising entry is undefined
+			}
+			else if (dt instanceof Undefined) {
+				// The new field either fully or partially contains preexisting fields
+				return;
+			}
+			offsetToDataTypeMap.subMap(firstEntry.getKey(), offset + dt.getLength()).clear();	// Clear overlapping entries
+		}
+		offsetToDataTypeMap.put(Long.valueOf(offset), dt);
+	}
+
+	/**
+	 * Adds information for a field given a pointer reference.
+	 * The data-type information is not used unless it is a pointer.
+	 * @param offset is the offset of the field within the structure
+	 * @param dt is the data-type of the pointer to the field (or null)
+	 */
+	public void addReference(long offset, DataType dt) {
+		if (dt != null && dt instanceof Pointer) {
+			dt = ((Pointer) dt).getDataType();
+			if (dt.equals(structDT)) {
+				return;		// Don't allow structure to contain itself
+			}
+			if (dt instanceof Structure) {
+				if (((Structure) dt).getNumDefinedComponents() == 0) {
+					computeMax(offset, 1);
+					return;
+				}
+			}
+			addDataType(offset, dt);
+		}
+		else {
+			computeMax(offset, 1);
+		}
+	}
+
+	/**
+	 * We may have partial information about the size of the structure.  This method feeds it to the
+	 * builder as a minimum size for the structure.
+	 * @param size is the minimum size in bytes
+	 */
+	public void setMinimumSize(long size) {
+		if (size > sizeOfStruct) {
+			sizeOfStruct = size;
+		}
+	}
+
+	/**
+	 * @return an iterator to the current field entries
+	 */
+	public Iterator<Entry<Long, DataType>> iterator() {
+		return offsetToDataTypeMap.entrySet().iterator();
+	}
+
+	/**
+	 * Populate this builder with fields from a preexisting Structure.
+	 * The builder presumes it is rebuilding this Structure so it can check for
+	 * pathological containment issues.
+	 * @param dt is the preexisting Structure
+	 */
+	public void populateOriginalStructure(Structure dt) {
+		structDT = dt;
+		DataTypeComponent[] components = structDT.getDefinedComponents();
+		for (DataTypeComponent component : components) {
+			offsetToDataTypeMap.put((long) component.getOffset(), component.getDataType());
+		}
+		sizeOfStruct = structDT.getLength();
+	}
+}

Ghidra/Framework/SoftwareModeling/src/test/java/ghidra/program/model/data/NoisyStructureBuilderTest.java

@@ -0,0 +1,101 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.program.model.data;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import generic.test.AbstractGTest;
+
+public class NoisyStructureBuilderTest extends AbstractGTest {
+
+	public void testNextField(Iterator<Entry<Long, DataType>> iter, long offset, DataType dt) {
+		Assert.assertTrue(iter.hasNext());
+		Entry<Long, DataType> entry = iter.next();
+		Assert.assertEquals(offset, entry.getKey().longValue());
+		Assert.assertTrue(entry.getValue() == dt);
+	}
+
+	@Test
+	public void testBasicFields() {
+		NoisyStructureBuilder builder = new NoisyStructureBuilder();
+		builder.addDataType(12, Undefined4DataType.dataType);
+		Pointer ptr = new Pointer32DataType(DWordDataType.dataType);
+		builder.addReference(4, ptr);
+		builder.addDataType(18, ShortDataType.dataType);
+		builder.addReference(21, null);
+
+		Iterator<Entry<Long, DataType>> iter = builder.iterator();
+		testNextField(iter, 4, DWordDataType.dataType);
+		testNextField(iter, 12, Undefined4DataType.dataType);
+		testNextField(iter, 18, ShortDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+		Assert.assertEquals(builder.getSize(), 22);
+
+		builder.addDataType(12, DWordDataType.dataType);
+		builder.addDataType(4, Undefined4DataType.dataType);
+
+		iter = builder.iterator();
+		testNextField(iter, 4, DWordDataType.dataType);
+		testNextField(iter, 12, DWordDataType.dataType);
+		testNextField(iter, 18, ShortDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+		Assert.assertEquals(builder.getSize(), 22);
+	}
+
+	@Test
+	public void testOverlaps() {
+		NoisyStructureBuilder builder = new NoisyStructureBuilder();
+		builder.addDataType(0, DWordDataType.dataType);
+		builder.addDataType(4, ShortDataType.dataType);
+		builder.addDataType(0, Undefined8DataType.dataType);
+		Assert.assertEquals(builder.getSize(), 8);
+		Iterator<Entry<Long, DataType>> iter = builder.iterator();
+		testNextField(iter, 0, DWordDataType.dataType);
+		testNextField(iter, 4, ShortDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+		Assert.assertEquals(builder.getSize(), 8);	// Undefined8 should expand size even though field isn't taken
+
+		builder.addDataType(0, QWordDataType.dataType);		// Should replace everything
+		iter = builder.iterator();
+		testNextField(iter, 0, QWordDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+		Pointer ptr = new Pointer32DataType(DWordDataType.dataType);
+		builder.addDataType(6, ptr);						// Partial overlap, should replace existing
+		iter = builder.iterator();
+		testNextField(iter, 6, ptr);
+		Assert.assertFalse(iter.hasNext());
+		Assert.assertEquals(builder.getSize(), 10);
+
+		builder.addDataType(4, DWordDataType.dataType);		// Partial overlap, should replace
+		iter = builder.iterator();
+		testNextField(iter, 4, DWordDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+
+		builder = new NoisyStructureBuilder();
+		builder.addDataType(4, Undefined8DataType.dataType);
+		builder.addDataType(4, Undefined4DataType.dataType);
+		builder.addDataType(8, DWordDataType.dataType);
+		builder.addDataType(8, SignedDWordDataType.dataType);	// Less specific data-type
+		iter = builder.iterator();
+		testNextField(iter, 4, Undefined4DataType.dataType);
+		testNextField(iter, 8, DWordDataType.dataType);
+		Assert.assertFalse(iter.hasNext());
+	}
+}