GP-5038 Support for data-type recursion through typedef

Ghidra/Features/Decompiler/src/decompile/cpp/type.cc

@@ -30,7 +30,7 @@ AttributeId ATTRIB_ARRAYSIZE = AttributeId("arraysize",48);
 AttributeId ATTRIB_CHAR = AttributeId("char",49);
 AttributeId ATTRIB_CORE = AttributeId("core",50);
 AttributeId ATTRIB_ENUM = AttributeId("enum",51);
-//AttributeId ATTRIB_ENUMSIGNED = AttributeId("enumsigned",52);  // deprecated
+AttributeId ATTRIB_INCOMPLETE = AttributeId("incomplete",52);
 //AttributeId ATTRIB_ENUMSIZE = AttributeId("enumsize",53);  // deprecated
 //AttributeId ATTRIB_INTSIZE = AttributeId("intsize",54);  // deprecated
 //AttributeId ATTRIB_LONGSIZE = AttributeId("longsize",55);  // deprecated
@@ -647,6 +647,10 @@ void Datatype::decodeBasic(Decoder &decoder)
     else if (attrib == ATTRIB_LABEL) {
       displayName = decoder.readString();
     }
+    else if (attrib == ATTRIB_INCOMPLETE) {
+      if (decoder.readBool())
+	flags |= type_incomplete;
+    }
   }
   if (size < 0)
     throw LowlevelError("Bad size for type "+name);
@@ -1885,10 +1889,10 @@ string TypeStruct::decodeFields(Decoder &decoder,TypeFactory &typegrp)
     if (curAlign > calcAlign)
       calcAlign = curAlign;
   }
-  if (size == 0)		// We can decode an incomplete structure, indicated by 0 size
+  if (size == 0)		// Old way to indicate an incomplete structure
-    flags |=  type_incomplete;
+    flags |= type_incomplete;
-  else
+  if (field.size() > 0)
-    markComplete();		// Otherwise the structure is complete
+    markComplete();		// If we have fields, mark as complete
   if (field.size() == 1) {			// A single field
     if (field[0].type->getSize() == size)	// that fills the whole structure
       flags |= needs_resolution;		// needs special resolution
@@ -2043,10 +2047,10 @@ void TypeUnion::decodeFields(Decoder &decoder,TypeFactory &typegrp)
     if (curAlign > calcAlign)
       calcAlign = curAlign;
   }
-  if (size == 0)		// We can decode an incomplete structure, indicated by 0 size
+  if (size == 0)		// Old way to indicate union is incomplete
-    flags |=  type_incomplete;
+    flags |= type_incomplete;
-  else
+  if (field.size() > 0)
-    markComplete();		// Otherwise the union is complete
+    markComplete();		// If we have fields, the union is complete
   if (alignment < 1)
     alignment = calcAlign;
   alignSize = calcAlignSize(size,alignment);
@@ -3188,6 +3192,7 @@ void TypeFactory::clear(void)
   nametree.clear();
   clearCache();
   warnings.clear();
+  incompleteTypedef.clear();
 }
 
 /// Delete anything that isn't a core type
@@ -3209,6 +3214,7 @@ void TypeFactory::clearNoncore(void)
     delete ct;
   }
   warnings.clear();
+  incompleteTypedef.clear();
 }
 
 TypeFactory::~TypeFactory(void)
@@ -3698,6 +3704,42 @@ void TypeFactory::removeWarning(Datatype *dt)
   }
 }
 
+/// Run through typedefs that were initially defined on incomplete data-types.  If the data-type is now complete,
+/// copy the fields or prototype into the typedef and remove it from the list.
+void TypeFactory::resolveIncompleteTypedefs(void)
+
+{
+  list<Datatype *>::iterator iter = incompleteTypedef.begin();
+  while(iter != incompleteTypedef.end()) {
+    Datatype *dt = *iter;
+    Datatype *defedType = dt->getTypedef();
+    if (!defedType->isIncomplete()) {
+      if (dt->getMetatype() == TYPE_STRUCT) {
+  	TypeStruct *prevStruct = (TypeStruct *)dt;
+  	TypeStruct *defedStruct = (TypeStruct *)defedType;
+  	setFields(defedStruct->field,prevStruct,defedStruct->size,defedStruct->alignment,defedStruct->flags);
+  	iter = incompleteTypedef.erase(iter);
+      }
+      else if (dt->getMetatype() == TYPE_UNION) {
+  	TypeUnion *prevUnion = (TypeUnion *)dt;
+  	TypeUnion *defedUnion = (TypeUnion *)defedType;
+  	setFields(defedUnion->field,prevUnion,defedUnion->size,defedUnion->alignment,defedUnion->flags);
+  	iter = incompleteTypedef.erase(iter);
+      }
+      else if (dt->getMetatype() == TYPE_CODE) {
+	TypeCode *prevCode = (TypeCode *)dt;
+	TypeCode *defedCode = (TypeCode *)defedType;
+	setPrototype(defedCode->proto, prevCode, defedCode->flags);
+	iter = incompleteTypedef.erase(iter);
+      }
+      else
+	++iter;
+    }
+    else
+      ++iter;
+  }
+}
+
 /// Find or create a data-type identical to the given data-type except for its name and id.
 /// If the name and id already describe an incompatible data-type, an exception is thrown.
 /// \param ct is the given data-type to clone
@@ -3724,6 +3766,8 @@ Datatype *TypeFactory::getTypedef(Datatype *ct,const string &name,uint8 id,uint4
   res->typedefImm = ct;
   res->setDisplayFormat(format);
   insert(res);
+  if (res->isIncomplete())
+    incompleteTypedef.push_back(res);
   return res;
 }
 
@@ -4234,6 +4278,7 @@ Datatype* TypeFactory::decodeStruct(Decoder &decoder,bool forcecore)
   }
   if (!warning.empty())
     insertWarning(ct, warning);
+  resolveIncompleteTypedefs();
 //  decoder.closeElement(elemId);
   return ct;
 }
@@ -4264,6 +4309,7 @@ Datatype* TypeFactory::decodeUnion(Decoder &decoder,bool forcecore)
   else {		// If structure is a placeholder stub
     setFields(tu.field,(TypeUnion*)ct,tu.size,tu.alignment,tu.flags);	// Define structure now by copying fields
   }
+  resolveIncompleteTypedefs();
 //  decoder.closeElement(elemId);
   return ct;
 }
@@ -4299,6 +4345,7 @@ Datatype *TypeFactory::decodeCode(Decoder &decoder,bool isConstructor,bool isDes
   else {	// If there was a placeholder stub
     setPrototype(tc.proto, (TypeCode *)ct, tc.flags);
   }
+  resolveIncompleteTypedefs();
 //  decoder.closeElement(elemId);
   return ct;
 }

Ghidra/Features/Decompiler/src/decompile/cpp/type.hh

@@ -28,7 +28,7 @@ extern AttributeId ATTRIB_ARRAYSIZE;	///< Marshaling attribute "arraysize"
 extern AttributeId ATTRIB_CHAR;		///< Marshaling attribute "char"
 extern AttributeId ATTRIB_CORE;		///< Marshaling attribute "core"
 extern AttributeId ATTRIB_ENUM;		///< Marshaling attribute "enum"
-//extern AttributeId ATTRIB_ENUMSIGNED;	///< Marshaling attribute "enumsigned" deprecated
+extern AttributeId ATTRIB_INCOMPLETE;	///< Marshaling attribute "incomplete"
 //extern AttributeId ATTRIB_ENUMSIZE;	///< Marshaling attribute "enumsize" deprecated
 //extern AttributeId ATTRIB_INTSIZE;	///< Marshaling attribute "intsize"  deprecated
 //extern AttributeId ATTRIB_LONGSIZE;	///< Marshaling attribute "longsize" deprecated
@@ -738,6 +738,7 @@ class TypeFactory {
   Datatype *type_nochar;	///< Same dimensions as char but acts and displays as an INT
   Datatype *charcache[5];	///< Cached character data-types
   list<DatatypeWarning> warnings;	///< Warnings for the user about data-types in \b this factory
+  list<Datatype *> incompleteTypedef;	///< Incomplete data-types defined as a \e typedef
   Datatype *findNoName(Datatype &ct);	///< Find data-type (in this container) by function
   void insert(Datatype *newtype);	///< Insert pointer into the cross-reference sets
   Datatype *findAdd(Datatype &ct);	///< Find data-type in this container or add it
@@ -757,6 +758,7 @@ class TypeFactory {
   void recalcPointerSubmeta(Datatype *base,sub_metatype sub);	///< Recalculate submeta for pointers to given base data-type
   void insertWarning(Datatype *dt,string warn);	///< Register a new data-type warning with \b this factory
   void removeWarning(Datatype *dt);		///< Remove the warning associated with the given data-type
+  void resolveIncompleteTypedefs(void);		///< Redefine incomplete typedefs of data-types that are now complete
 protected:
   Architecture *glb;		///< The Architecture object that owns this TypeFactory
   Datatype *findByIdLocal(const string &nm,uint8 id) const;	///< Search locally by name and id

Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileDebug.java

@@ -340,7 +340,7 @@ public class DecompileDebug {
 			new DataTypeDependencyOrderer(program.getDataTypeManager(), dtypes);
 		//First output all structures as zero size so to avoid any cyclic dependencies.
 		for (DataType dataType : TypeOrderer.getCompositeList()) {
-			dtmanage.encodeCompositeZeroSizePlaceholder(encoder, dataType);
+			dtmanage.encodeCompositePlaceholder(encoder, dataType);
 		}
 		//Next, use the dependency stack to output types.
 		for (DataType dataType : TypeOrderer.getDependencyList()) {

Ghidra/Features/Decompiler/src/main/java/ghidra/app/util/DataTypeDependencyOrderer.java

@@ -337,8 +337,9 @@ public class DataTypeDependencyOrderer {
 		if (!doneSet.contains(subEntry)) {
 			procSet.add(subEntry);
 		}
-		if (entry.dataType instanceof Pointer) { //avoid cycles with structures/composites
+		if (subType instanceof Composite) {
-			if (subType instanceof Composite) {
+			// Trim dependency from pointer or typedef to a composite to prevent cycles
+			if (entry.dataType instanceof Pointer || entry.dataType instanceof TypeDef) {
 				return;
 			}
 		}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/AttributeId.java

@@ -115,7 +115,7 @@ public record AttributeId(String name, int id) {
 	public static final AttributeId ATTRIB_CHAR = new AttributeId("char", 49);
 	public static final AttributeId ATTRIB_CORE = new AttributeId("core", 50);
 	public static final AttributeId ATTRIB_ENUM = new AttributeId("enum", 51);
-//	public static final AttributeId ATTRIB_ENUMSIGNED = new AttributeId("enumsigned", 52);  // deprecated
+	public static final AttributeId ATTRIB_INCOMPLETE = new AttributeId("incomplete", 52);
 //	public static final AttributeId ATTRIB_ENUMSIZE = new AttributeId("enumsize", 53);  // deprecated
 //	public static final AttributeId ATTRIB_INTSIZE = new AttributeId("intsize", 54);  // deprecated
 //	public static final AttributeId ATTRIB_LONGSIZE = new AttributeId("longsize", 55);  // deprecated

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/PcodeDataTypeManager.java

@@ -799,12 +799,12 @@ public class PcodeDataTypeManager {
 	}
 
 	/**
-	 * Encode a Structure to the stream that has its size reported as zero.
+	 * Encode a Structure/Union to the stream without listing its fields
 	 * @param encoder is the stream encoder
 	 * @param type data type to encode
 	 * @throws IOException for errors in the underlying stream
 	 */
-	public void encodeCompositeZeroSizePlaceholder(Encoder encoder, DataType type)
+	public void encodeCompositePlaceholder(Encoder encoder, DataType type)
 			throws IOException {
 		String metaString;
 		if (type instanceof Structure) {
@@ -820,7 +820,9 @@ public class PcodeDataTypeManager {
 		encoder.writeString(ATTRIB_NAME, type.getDisplayName());
 		encoder.writeUnsignedInteger(ATTRIB_ID, progDataTypes.getID(type));
 		encoder.writeString(ATTRIB_METATYPE, metaString);
-		encoder.writeSignedInteger(ATTRIB_SIZE, 0);
+		encoder.writeSignedInteger(ATTRIB_SIZE, type.getLength());
+		encoder.writeSignedInteger(ATTRIB_ALIGNMENT, type.getAlignment());
+		encoder.writeBool(ATTRIB_INCOMPLETE, true);
 		encoder.closeElement(ELEM_TYPE);
 	}