Opaque strings and variable length data-types

Ghidra/Features/Decompiler/src/decompile/cpp/cast.cc

@@ -234,7 +234,12 @@ Datatype *CastStrategyC::castStandard(Datatype *reqtype,Datatype *curtype,
   if (curtype == reqtype) return (Datatype *)0;	// Different typedefs could point to the same type
   if ((reqbase->getMetatype()==TYPE_VOID)||(reqbase->getMetatype()==TYPE_VOID))
     return (Datatype *)0;	// Don't cast from or to VOID
-  if (reqbase->getSize() != curbase->getSize()) return reqtype; // Always cast change in size
+  if (reqbase->getSize() != curbase->getSize()) {
+    if (reqbase->isVariableLength() && isptr && reqbase->hasSameVariableBase(curbase)) {
+      return (Datatype *)0;	// Don't need a cast
+    }
+    return reqtype; // Otherwise, always cast change in size
+  }
   switch(reqbase->getMetatype()) {
   case TYPE_UNKNOWN:
     return (Datatype *)0;

Ghidra/Features/Decompiler/src/decompile/cpp/printc.cc

@@ -1223,7 +1223,7 @@ bool PrintC::printCharacterConstant(ostream &s,const Address &addr,Datatype *cha
   const vector<uint1> &buffer(manager->getStringData(addr, charType, isTrunc));
   if (buffer.empty())
     return false;
-  if (doEmitWideCharPrefix() && charType->getSize() > 1)
+  if (doEmitWideCharPrefix() && charType->getSize() > 1 && !charType->isOpaqueString())
     s << 'L';			// Print symbol indicating wide character
   s << '"';
   escapeCharacterData(s,buffer.data(),buffer.size(),1,glb->translate->isBigEndian());

Ghidra/Features/Decompiler/src/decompile/cpp/type.cc

@@ -63,6 +63,20 @@ void print_data(ostream &s,uint1 *buffer,int4 size,const Address &baseaddr)
   }
 }
 
+/// If \b this and the other given data-type are both variable length and come from the
+/// the same base data-type, return \b true.
+/// \param ct is the other given data-type to compare with \b this
+/// \return \b true if they are the same variable length data-type.
+bool Datatype::hasSameVariableBase(const Datatype *ct) const
+
+{
+  if (!isVariableLength()) return false;
+  if (!ct->isVariableLength()) return false;
+  uint8 thisId = hashSize(id, size);
+  uint8 themId = hashSize(ct->id, ct->size);
+  return (thisId == themId);
+}
+
 /// Print a raw description of the type to stream. Intended for debugging.
 /// Not intended to produce parsable C.
 /// \param s is the output stream
@@ -240,8 +254,13 @@ void Datatype::saveXmlBasic(ostream &s) const
 
 {
   a_v(s,"name",name);
-  if (id != 0) {
-    s << " id=\"0x" << hex << id << '\"';
+  uint8 saveId;
+  if (isVariableLength())
+    saveId = hashSize(id, size);
+  else
+    saveId = id;
+  if (saveId != 0) {
+    s << " id=\"0x" << hex << saveId << '\"';
   }
   a_v_i(s,"size",size);
   string metastring;
@@ -249,6 +268,10 @@ void Datatype::saveXmlBasic(ostream &s) const
   a_v(s,"metatype",metastring);
   if ((flags & coretype)!=0)
     a_v_b(s,"core",true);
+  if (isVariableLength())
+    a_v_b(s,"varlength",true);
+  if ((flags & opaque_string)!=0)
+    a_v_b(s,"opaquestring",true);
 }
 
 /// Write a simple reference to \b this data-type as an XML \<typeref> tag,
@@ -283,18 +306,31 @@ void Datatype::restoreXmlBasic(const Element *el)
   metatype = string2metatype( el->getAttributeValue("metatype") );
   id = 0;
   for(int4 i=0;i<el->getNumAttributes();++i) {
-    if (el->getAttributeName(i) == "core") {
+    const string &attribName( el->getAttributeName(i) );
+    if (attribName == "core") {
       if (xml_readbool(el->getAttributeValue(i)))
 	flags |= coretype;
     }
-    else if (el->getAttributeName(i) == "id") {
+    else if (attribName == "id") {
       istringstream i1(el->getAttributeValue(i));
       i1.unsetf(ios::dec | ios::hex | ios::oct);
       i1 >> id;
     }
+    else if (attribName == "varlength") {
+      if (xml_readbool(el->getAttributeValue(i)))
+	flags |= variable_length;
+    }
+    else if (attribName == "opaquestring") {
+      if (xml_readbool(el->getAttributeValue(i)))
+	flags |= opaque_string;
+    }
   }
   if ((id==0)&&(name.size()>0))	// If there is a type name
     id = hashName(name);	// There must be some kind of id
+  if (isVariableLength()) {
+    // Id needs to be unique compared to another data-type with the same name
+    id = hashSize(id, size);
+  }
 }
 
 /// Restore a Datatype object from an XML element
@@ -326,6 +362,21 @@ uint8 Datatype::hashName(const string &nm)
   return res;
 }
 
+/// This allows IDs for variable length structures to be uniquefied based on size.
+/// A base ID is given and a size of the specific instance. A unique ID is returned.
+/// The hashing is reversible by feeding the output ID back into this function with the same size.
+/// \param id is the given ID to (de)uniquify
+/// \param size is the instance size of the structure
+/// \param return the (de)uniquified id
+uint8 Datatype::hashSize(uint8 id,int4 size)
+
+{
+  uint8 sizeHash = size;
+  sizeHash *= 0x98251033aecbabaf;	// Hash the size
+  id ^= sizeHash;
+  return id;
+}
+
 void TypeChar::saveXml(ostream &s) const
 
 {
@@ -1483,8 +1534,9 @@ Datatype *TypeFactory::setName(Datatype *ct,const string &n)
 /// \param fd is the list of fields to set
 /// \param ot is the TypeStruct object to modify
 /// \param fixedsize is 0 or the forced size of the structure
+/// \param flags are other flags to set on the structure
 /// \return true if modification was successful
-bool TypeFactory::setFields(vector<TypeField> &fd,TypeStruct *ot,int4 fixedsize)
+bool TypeFactory::setFields(vector<TypeField> &fd,TypeStruct *ot,int4 fixedsize,uint4 flags)
 
 {
   int4 offset,cursize,curalign;
@@ -1529,6 +1581,7 @@ bool TypeFactory::setFields(vector<TypeField> &fd,TypeStruct *ot,int4 fixedsize)
 
   tree.erase(ot);
   ot->setFields(fd);
+  ot->flags |= (flags & (Datatype::opaque_string | Datatype::variable_length));
   if (fixedsize > 0) {		// If the caller is trying to force a size
     if (fixedsize > ot->size)	// If the forced size is bigger than the size required for fields
       ot->size = fixedsize;	//     Force the bigger size
@@ -2073,20 +2126,27 @@ Datatype *TypeFactory::restoreXmlTypeNoRef(const Element *el,bool forcecore)
       int4 num = el->getNumAttributes();
       uint8 newid = 0;
       int4 structsize = 0;
+      bool isVarLength = false;
       for(int4 i=0;i<num;++i) {
-	if (el->getAttributeName(i) == "id") {
+	const string &attribName(el->getAttributeName(i));
+	if (attribName == "id") {
 	  istringstream s(el->getAttributeValue(i));
 	  s.unsetf(ios::dec | ios::hex | ios::oct);
 	  s >> newid;
 	}
-	else if (el->getAttributeName(i) == "size") {
+	else if (attribName == "size") {
 	  istringstream s(el->getAttributeValue(i));
 	  s.unsetf(ios::dec | ios::hex | ios::oct);
 	  s >> structsize;
 	}
+	else if (attribName == "varlength") {
+	  isVarLength = xml_readbool(el->getAttributeValue(i));
+	}
       }
       if (newid == 0)
 	newid = Datatype::hashName(structname);
+      if (isVarLength)
+	newid = Datatype::hashSize(newid, structsize);
       ct = findByIdLocal(structname,newid);
       bool stubfirst = false;
       if (ct == (Datatype *)0) {
@@ -2105,7 +2165,7 @@ Datatype *TypeFactory::restoreXmlTypeNoRef(const Element *el,bool forcecore)
 	  throw LowlevelError("Redefinition of structure: "+structname);
       }
       else			// If structure is a placeholder stub
-	if (!setFields(ts.field,(TypeStruct *)ct,ts.size)) // Define structure now by copying fields
+	if (!setFields(ts.field,(TypeStruct *)ct,ts.size,ts.flags)) // Define structure now by copying fields
 	  throw LowlevelError("Bad structure definition");
     }
     break;

Ghidra/Features/Decompiler/src/decompile/cpp/type.hh

@@ -73,7 +73,9 @@ protected:
     enumtype = 4,		///< An enumeration type (as well as an integer)
     poweroftwo = 8,		///< An enumeration type where all values are of 2^^n form
     utf16 = 16,			///< 16-bit wide chars in unicode UTF16
-    utf32 = 32			///< 32-bit wide chars in unicode UTF32
+    utf32 = 32,			///< 32-bit wide chars in unicode UTF32
+    opaque_string = 64,		///< Structure that should be treated as a string
+    variable_length = 128	///< May be other structures with same name different lengths
   };
   friend class TypeFactory;
   friend struct DatatypeCompare;
@@ -85,6 +87,7 @@ protected:
   void restoreXmlBasic(const Element *el);	///< Recover basic data-type properties
   virtual void restoreXml(const Element *el,TypeFactory &typegrp);	///< Restore data-type from XML
   static uint8 hashName(const string &nm);	///< Produce a data-type id by hashing the type name
+  static uint8 hashSize(uint8 id,int4 size);	///< Reversibly hash size into id
 public:
   /// Construct the base data-type copying low-level properties of another
   Datatype(const Datatype &op) { size = op.size; name=op.name; metatype=op.metatype; flags=op.flags; id=op.id; }
@@ -94,12 +97,15 @@ public:
   Datatype(int4 s,type_metatype m,const string &n) { name=n; size=s; metatype=m; flags=0; id=0; }
   virtual ~Datatype(void) {}	///< Destructor
   bool isCoreType(void) const { return ((flags&coretype)!=0); }	///< Is this a core data-type
-  bool isCharPrint(void) const { return ((flags&(chartype|utf16|utf32))!=0); }	///< Does this print as a 'char'
+  bool isCharPrint(void) const { return ((flags&(chartype|utf16|utf32|opaque_string))!=0); }	///< Does this print as a 'char'
   bool isEnumType(void) const { return ((flags&enumtype)!=0); }		///< Is this an enumerated type
   bool isPowerOfTwo(void) const { return ((flags&poweroftwo)!=0); }	///< Is this a flag-based enumeration
   bool isASCII(void) const { return ((flags&chartype)!=0); }	///< Does this print as an ASCII 'char'
   bool isUTF16(void) const { return ((flags&utf16)!=0); }	///< Does this print as UTF16 'wchar'
   bool isUTF32(void) const { return ((flags&utf32)!=0); }	///< Does this print as UTF32 'wchar'
+  bool isVariableLength(void) const { return ((flags&variable_length)!=0); }	///< Is \b this a variable length structure
+  bool hasSameVariableBase(const Datatype *ct) const;		///< Are these the same variable length data-type
+  bool isOpaqueString(void) const { return ((flags&opaque_string)!=0); }	///< Is \b this an opaquely encoded string
   uint4 getInheritable(void) const { return (flags & coretype); }	///< Get properties pointers inherit
   type_metatype getMetatype(void) const { return metatype; }	///< Get the type \b meta-type
   uint8 getId(void) const { return id; }			///< Get the type id
@@ -412,7 +418,7 @@ public:
   Architecture *getArch(void) const { return glb; }	///< Get the Architecture object
   Datatype *findByName(const string &n);		///< Return type of given name
   Datatype *setName(Datatype *ct,const string &n); 	///< Set the given types name
-  bool setFields(vector<TypeField> &fd,TypeStruct *ot,int4 fixedsize);	///< Set fields on a TypeStruct
+  bool setFields(vector<TypeField> &fd,TypeStruct *ot,int4 fixedsize,uint4 flags);	///< Set fields on a TypeStruct
   bool setEnumValues(const vector<string> &namelist,
 		      const vector<uintb> &vallist,
 		      const vector<bool> &assignlist,