GP-3126 Join support for CONCAT trees

Ghidra/Features/Decompiler/certification.manifest

@@ -49,6 +49,7 @@ src/decompile/datatests/pointerrel.xml||GHIDRA||||END|
 src/decompile/datatests/pointersub.xml||GHIDRA||||END|
 src/decompile/datatests/promotecompare.xml||GHIDRA||||END|
 src/decompile/datatests/readvolatile.xml||GHIDRA||||END|
+src/decompile/datatests/retstruct.xml||GHIDRA||||END|
 src/decompile/datatests/sbyte.xml||GHIDRA||||END|
 src/decompile/datatests/skipnext2.xml||GHIDRA||||END|
 src/decompile/datatests/statuscmp.xml||GHIDRA||||END|

Ghidra/Features/Decompiler/src/decompile/cpp/address.cc

@@ -162,6 +162,32 @@ int4 Address::overlap(int4 skip,const Address &op,int4 size) const
   return (int4) dist;
 }
 
+/// This method is equivalent to Address::overlap, but a range in the \e join space can be
+/// considered overlapped with its constituent pieces.
+/// If \e this + \e skip falls in the range, \e op to \e op + \e size, then a non-negative integer is
+/// returned indicating where in the interval it falls. Otherwise -1 is returned.
+/// \param skip is an adjust to \e this address
+/// \param op is the start of the range to check
+/// \param size is the size of the range
+/// \return an integer indicating how overlap occurs
+int4 Address::overlapJoin(int4 skip,const Address &op,int4 size) const
+
+{
+  if (base != op.base) {
+    if (op.base->getType()==IPTR_JOIN) {
+      uintb dist = base->wrapOffset(offset + skip);
+      return ((JoinSpace *)op.base)->pieceOverlap(op.offset,size,base,dist);
+    }
+    return -1;
+  }
+  if (base->getType()==IPTR_CONSTANT) return -1; // Must not be constants
+
+  uintb dist = base->wrapOffset(offset+skip-op.offset);
+
+  if (dist >= size) return -1; // but must fall before op+size
+  return (int4) dist;
+}
+
 /// Does the location \e this, \e sz form a contiguous region to \e loaddr, \e losz,
 /// where \e this forms the most significant piece of the logical whole
 /// \param sz is the size of \e this hi region
@@ -184,7 +210,7 @@ bool Address::isContiguous(int4 sz,const Address &loaddr,int4 losz) const
 }
 
 /// If \b this is (originally) a \e join address, reevaluate it in terms of its new
-/// \e offset and \e siz, changing the space and offset if necessary.
+/// \e offset and \e size, changing the space and offset if necessary.
 /// \param size is the new size in bytes of the underlying object
 void Address::renormalize(int4 size) {
   if (base->getType() == IPTR_JOIN)

Ghidra/Features/Decompiler/src/decompile/cpp/address.hh

@@ -88,6 +88,7 @@ public:
   bool containedBy(int4 sz,const Address &op2,int4 sz2) const;	///< Determine if \e op2 range contains \b this range
   int4 justifiedContain(int4 sz,const Address &op2,int4 sz2,bool forceleft) const; ///< Determine if \e op2 is the least significant part of \e this.
   int4 overlap(int4 skip,const Address &op,int4 size) const; ///< Determine how two address ranges overlap
+  int4 overlapJoin(int4 skip,const Address &op,int4 size) const;	///< Determine how two ranges overlap, when one might be in the \e join space
   bool isContiguous(int4 sz,const Address &loaddr,int4 losz) const; ///< Does \e this form a contigous range with \e loaddr
   bool isConstant(void) const; ///< Is this a \e constant \e value
   void renormalize(int4 size);	///< Make sure there is a backing JoinRecord if \b this is in the \e join space

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc

@@ -2850,7 +2850,7 @@ int4 ActionMarkExplicit::baseExplicit(Varnode *vn,int4 maxref)
     if (def->code() == CPUI_SUBPIECE) {
       Varnode *vin = def->getIn(0);
       if (vin->isAddrTied()) {
-        if (vn->overlap(*vin) == def->getIn(1)->getOffset())
+        if (vn->overlapJoin(*vin) == def->getIn(1)->getOffset())
           return -1;		// Should be explicit, will be a copymarker and not printed
       }
     }

Ghidra/Features/Decompiler/src/decompile/cpp/ifacedecomp.cc

@@ -114,6 +114,7 @@ void IfaceDecompCapability::registerCommands(IfaceStatus *status)
   status->registerCom(new IfcRename(),"rename");
   status->registerCom(new IfcRetype(),"retype");
   status->registerCom(new IfcRemove(),"remove");
+  status->registerCom(new IfcIsolate(),"isolate");
   status->registerCom(new IfcLockPrototype(),"prototype","lock");
   status->registerCom(new IfcUnlockPrototype(),"prototype","unlock");
   status->registerCom(new IfcCommentInstr(),"comment","instruction");
@@ -1365,6 +1366,29 @@ void IfcRetype::execute(istream &s)
   }
 }
 
+/// \class IfcIsolate
+/// \brief Mark a symbol as isolated from speculative merging: `isolate <name>`
+void IfcIsolate::execute(istream &s)
+
+{
+  string symbolName;
+
+  s >> ws >> symbolName;
+  if (symbolName.size() == 0)
+    throw IfaceParseError("Missing symbol name");
+
+  Symbol *sym;
+  vector<Symbol *> symList;
+  dcp->readSymbol(symbolName,symList);
+  if (symList.empty())
+    throw IfaceExecutionError("No symbol named: "+symbolName);
+  if (symList.size() == 1)
+    sym = symList[0];
+  else
+    throw IfaceExecutionError("More than one symbol named: "+symbolName);
+  sym->setIsolated(true);
+}
+
 /// The Varnode is selected from the \e current function.  It is specified as a
 /// storage location with info about its defining p-code in parantheses.
 ///   - `%EAX(r0x10000:0x65)`

Ghidra/Features/Decompiler/src/decompile/cpp/ifacedecomp.hh

@@ -323,6 +323,11 @@ public:
   virtual void execute(istream &s);
 };
 
+class IfcIsolate : public IfaceDecompCommand {
+public:
+  virtual void execute(istream &s);
+};
+
 class IfcPrintVarnode : public IfaceDecompCommand {
 public:
   virtual void execute(istream &s);

Ghidra/Features/Decompiler/src/decompile/cpp/merge.cc

@@ -103,6 +103,18 @@ bool Merge::mergeTestRequired(HighVariable *high_out,HighVariable *high_in)
   else if (high_out->isExtraOut())
     return false;
 
+  if (high_in->isProtoPartial()) {
+    if (high_out->isProtoPartial()) return false;
+    if (high_out->isInput()) return false;
+    if (high_out->isAddrTied()) return false;
+    if (high_out->isPersist()) return false;
+  }
+  if (high_out->isProtoPartial()) {
+    if (high_in->isInput()) return false;
+    if (high_in->isAddrTied()) return false;
+    if (high_in->isPersist()) return false;
+  }
+
   Symbol *symbolIn = high_in->getSymbol();
   Symbol *symbolOut = high_out->getSymbol();
   if (symbolIn != (Symbol *) 0 && symbolOut != (Symbol *) 0) {
@@ -1456,10 +1468,10 @@ void Merge::markInternalCopies(void)
       h1 = op->getOut()->getHigh();
       h2 = op->getIn(0)->getHigh();
       h3 = op->getIn(1)->getHigh();
-      if (!h2->isPartial()) break;
+      if (!h2->isPartialOrAddrTied()) break;
-      if (!h3->isPartial()) break;
+      if (!h3->isPartialOrAddrTied()) break;
-      v2 = h2->getPartial();
+      v2 = h2->getPartialOrAddrTied();
-      v3 = h3->getPartial();
+      v3 = h3->getPartialOrAddrTied();
       if (v2->isAddrTied()) {
 	if (!h1->isAddrTied()) break;
 	v1 = h1->getTiedVarnode();
@@ -1469,15 +1481,15 @@ void Merge::markInternalCopies(void)
 	if (op->getIn(1) != v3) break;
 	v1 = op->getOut();
       }
-      if (v3->overlap(*v1) != 0) break;
+      if (v3->overlapJoin(*v1) != 0) break;
-      if (v2->overlap(*v1) != v3->getSize()) break;
+      if (v2->overlapJoin(*v1) != v3->getSize()) break;
       data.opMarkNonPrinting(op);
       break;
     case CPUI_SUBPIECE:
       h1 = op->getOut()->getHigh();
       h2 = op->getIn(0)->getHigh();
-      if (!h1->isPartial()) break;
+      if (!h1->isPartialOrAddrTied()) break;
-      v1 = h1->getPartial();
+      v1 = h1->getPartialOrAddrTied();
       if (v1->isAddrTied()) {
 	if (!h2->isAddrTied()) break;
 	v2 = h2->getTiedVarnode();
@@ -1487,7 +1499,7 @@ void Merge::markInternalCopies(void)
 	v2 = op->getIn(0);
       }
       val = op->getIn(1)->getOffset();
-      if (v1->overlap(*v2) != val) break;
+      if (v1->overlapJoin(*v2) != val) break;
       data.opMarkNonPrinting(op);
       break;
     default:

Ghidra/Features/Decompiler/src/decompile/cpp/op.cc

@@ -796,6 +796,7 @@ Varnode *PieceNode::findRoot(Varnode *vn)
       Address addr = op->getOut()->getAddr();
       if (addr.getSpace()->isBigEndian() == (slot == 1))
 	addr = addr + op->getIn(1-slot)->getSize();
+      addr.renormalize(vn->getSize());		// Allow for possible join address
       if (addr == vn->getAddr()) {
 	if (pieceOp != (PcodeOp *)0) {		// If there is more than one valid PIECE
 	  if (op->compareOrder(pieceOp))	// Attach this to earliest one

Ghidra/Features/Decompiler/src/decompile/cpp/ruleaction.cc

@@ -6965,6 +6965,7 @@ bool RulePieceStructure::separateSymbol(Varnode *root,Varnode *leaf)
   if (!leaf->isWritten()) return true;	// Assume to be different symbols
   if (leaf->isProtoPartial()) return true;	// Already in another tree
   PcodeOp *op = leaf->getDef();
+  if (op->isMarker()) return true;	// Leaf is not defined locally
   if (op->code() != CPUI_PIECE) return false;
   if (leaf->getType()->isPieceStructured()) return true;	// Would be a separate root
 
@@ -7025,6 +7026,7 @@ int4 RulePieceStructure::applyOp(PcodeOp *op,Funcdata &data)
     PieceNode &node(stack[i]);
     Varnode *vn = node.getVarnode();
     Address addr = baseAddr + node.getTypeOffset();
+    addr.renormalize(vn->getSize());		// Allow for possible join address
     if (vn->getAddr() == addr) {
       if (!node.isLeaf() || !separateSymbol(outvn, vn)) {
 	// Varnode already has correct address and will be part of the same symbol as root

Ghidra/Features/Decompiler/src/decompile/cpp/space.cc

@@ -471,6 +471,48 @@ JoinSpace::JoinSpace(AddrSpaceManager *m,const Translate *t,int4 ind)
   clearFlags(heritaged); // This space is never heritaged, but does dead-code analysis
 }
 
+/// \brief Calculate where a given address falls inside a range in the \e join space
+///
+/// The given address, specified as an (AddrSpace, offset) pair is assumed to not be in the \e join space,
+/// but is assumed to be in one of the pieces of the join.  A final non-negative offset is returned if the
+/// address falls in one of the pieces. The offset is in data order, i.e. the offset is 0 if the given Address
+/// matches the address of the first element in an array overlaid on the \e join range.
+/// -1 is returned if the given Address does not lie in any piece.
+/// \param offset is the offset into the \e join space, specifying the range
+/// \param size is the size of the range, which may be smaller than the size of the JoinRecord
+/// \param pieceSpace is the AddrSpace of the given address
+/// \param pieceOffset is the offset of the given address
+/// \return a non-negative offset indicating where in the range the Address lies, or -1
+int4 JoinSpace::pieceOverlap(uintb offset,int4 size,AddrSpace *pieceSpace,uintb pieceOffset) const
+
+{
+  JoinRecord *joinRecord = getManager()->findJoin(offset);
+  // Set up so we traverse pieces in data order
+  int4 startPiece,endPiece,dir;
+  if (isBigEndian()) {
+    startPiece = 0;
+    endPiece = joinRecord->numPieces();
+    dir = 1;
+  }
+  else {
+    startPiece = joinRecord->numPieces() - 1;
+    endPiece = -1;
+    dir = -1;
+  }
+  int4 bytesAccum = 0;
+  for(int4 i=startPiece;i!=endPiece;i += dir) {
+    const VarnodeData &vData(joinRecord->getPiece(i));
+    if (vData.space == pieceSpace && pieceOffset >= vData.offset && pieceOffset <= vData.offset + (vData.size-1)) {
+      int4 res = (int4)(pieceOffset - vData.offset) + bytesAccum;
+      if (res >= size)
+	return -1;
+      return res;
+    }
+    bytesAccum += vData.size;
+  }
+  return -1;
+}
+
 /// Encode a \e join address to the stream.  This method in the interface only
 /// outputs attributes for a single element, so we are forced to encode what should probably
 /// be recursive elements into an attribute.

Ghidra/Features/Decompiler/src/decompile/cpp/space.hh

@@ -240,6 +240,7 @@ public:
 class JoinSpace : public AddrSpace {
 public:
   JoinSpace(AddrSpaceManager *m,const Translate *t,int4 ind);
+  int4 pieceOverlap(uintb offset,int4 size,AddrSpace *pieceSpace,uintb pieceOffset) const;
   virtual void encodeAttributes(Encoder &encoder,uintb offset) const;
   virtual void encodeAttributes(Encoder &encoder,uintb offset,int4 size) const;
   virtual uintb decodeAttributes(Decoder &decoder,uint4 &size) const;

Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc

@@ -233,7 +233,7 @@ void HighVariable::setSymbol(Varnode *vn) const
     if (rootVn == vn)
       throw LowlevelError("Partial varnode does not match symbol");
 
-    symboloffset = vn->getAddr().overlap(0,rootVn->getAddr(),rootVn->getSize());
+    symboloffset = vn->getAddr().overlapJoin(0,rootVn->getAddr(),rootVn->getSize());
     SymbolEntry *entry = rootVn->getSymbolEntry();
     if (entry != (SymbolEntry *)0)
       symboloffset += entry->getOffset();
@@ -246,7 +246,7 @@ void HighVariable::setSymbol(Varnode *vn) const
       entry->getAddr() == vn->getAddr() && !entry->isPiece())
     symboloffset = -1;			// A matching entry
   else {
-    symboloffset = vn->getAddr().overlap(0,entry->getAddr(),symbol->getType()->getSize()) + entry->getOffset();
+    symboloffset = vn->getAddr().overlapJoin(0,entry->getAddr(),symbol->getType()->getSize()) + entry->getOffset();
   }
 
   highflags &= ~((uint4)symboldirty);		// We are no longer dirty
@@ -426,6 +426,8 @@ bool HighVariable::compareName(Varnode *vn1,Varnode *vn2)
     return vn2->isInput();
   if (vn1->isAddrTied() != vn2->isAddrTied()) // Prefer address tied
     return vn2->isAddrTied();
+  if (vn1->isProtoPartial() != vn2->isProtoPartial())	// Prefer pieces
+    return vn2->isProtoPartial();
 
   // Prefer NOT internal
   if ((vn1->getSpace()->getType() != IPTR_INTERNAL)&&
@@ -469,7 +471,7 @@ Varnode *HighVariable::getNameRepresentative(void) const
 
 /// Find the first member that is either address tied or marked as a proto partial.
 /// \return a member Varnode acting as partial storage or null if none exist
-Varnode *HighVariable::getPartial(void) const
+Varnode *HighVariable::getPartialOrAddrTied(void) const
 
 {
   int4 i;

Ghidra/Features/Decompiler/src/decompile/cpp/variable.hh

@@ -179,7 +179,7 @@ public:
   Varnode *getInputVarnode(void) const;		///< Find (the) input member Varnode
   Varnode *getTypeRepresentative(void) const;	///< Get a member Varnode with the strongest data-type
   Varnode *getNameRepresentative(void) const;	///< Get a member Varnode that dictates the naming of \b this HighVariable
-  Varnode *getPartial(void) const;		///< Find the first member that can act as partial symbol storage
+  Varnode *getPartialOrAddrTied(void) const;	///< Find the first member that can act as partial symbol storage
   int4 getNumMergeClasses(void) const { return numMergeClasses; }	///< Get the number of speculative merges for \b this
   bool isMapped(void) const { updateFlags(); return ((flags&Varnode::mapped)!=0); }	///< Return \b true if \b this is mapped
   bool isPersist(void) const { updateFlags(); return ((flags&Varnode::persist)!=0); }	///< Return \b true if \b this is a global variable
@@ -190,7 +190,8 @@ public:
   bool isConstant(void) const { updateFlags(); return ((flags&Varnode::constant)!=0); }	///< Return \b true if \b this is a constant
   bool isUnaffected(void) const { updateFlags(); return ((flags&Varnode::unaffected)!=0); }	///< Return \b true if \b this is an \e unaffected register
   bool isExtraOut(void) const { updateFlags(); return ((flags&(Varnode::indirect_creation|Varnode::addrtied))==Varnode::indirect_creation); }	///< Return \b true if \b this is an extra output
-  bool isPartial(void) const { updateFlags(); return ((flags&(Varnode::addrtied|Varnode::proto_partial))!=0); }	///< Return \b true if \b this is potential partial symbol
+  bool isProtoPartial(void) const { updateFlags(); return ((flags&Varnode::proto_partial)!=0); }	///< Return \b true if \b this is a piece concatenated into a larger whole
+  bool isPartialOrAddrTied(void) const { updateFlags(); return ((flags&(Varnode::addrtied|Varnode::proto_partial))!=0); }	///< Return \b true if \b this is potential partial symbol
   void setMark(void) const { flags |= Varnode::mark; }		///< Set the mark on this variable
   void clearMark(void) const { flags &= ~Varnode::mark; }	///< Clear the mark on this variable
   bool isMark(void) const { return ((flags&Varnode::mark)!=0); }	///< Return \b true if \b this is marked

Ghidra/Features/Decompiler/src/decompile/cpp/varnode.cc

@@ -171,7 +171,7 @@ int4 Varnode::characterizeOverlap(const Varnode &op) const
 /// I.e. return
 ///     - 0 if it overlaps op's lsb
 ///     - 1 if it overlaps op's second lsb  and so on
-/// \param op is Varnode to test for overlap
+/// \param op is the Varnode to test for overlap
 /// \return the relative overlap point or -1
 int4 Varnode::overlap(const Varnode &op) const
 
@@ -186,6 +186,25 @@ int4 Varnode::overlap(const Varnode &op) const
   return -1;
 }
 
+/// Return whether \e Least \e Signifigant \e Byte of \b this occurs in \b op.
+/// If \b op is in the \e join space, \b this can be in one of the pieces associated with the \e join range, and
+/// the offset returned will take into account the relative position of the piece within the whole \e join.
+/// Otherwise, this method is equivalent to Varnode::overlap.
+/// \param op is the Varnode to test for overlap
+/// \return the relative overlap point or -1
+int4 Varnode::overlapJoin(const Varnode &op) const
+
+{
+  if (!loc.isBigEndian())	// Little endian
+    return loc.overlapJoin(0,op.loc,op.size);
+  else {			// Big endian
+    int4 over = loc.overlapJoin(size-1,op.loc,op.size);
+    if (over != -1)
+      return op.size-1-over;
+  }
+  return -1;
+}
+
 /// Return whether \e Least \e Signifigant \e Byte of \b this occurs in an Address range
 /// I.e. return
 ///     - 0 if it overlaps op's lsb

Ghidra/Features/Decompiler/src/decompile/cpp/varnode.hh

@@ -223,6 +223,7 @@ public:
   int4 contains(const Varnode &op) const; ///< Return info about the containment of \e op in \b this
   int4 characterizeOverlap(const Varnode &op) const; ///< Return 0, 1, or 2 for "no overlap", "partial overlap", "identical storage"
   int4 overlap(const Varnode &op) const;	///< Return relative point of overlap between two Varnodes
+  int4 overlapJoin(const Varnode &op) const;	///< Return relative point of overlap, where the given Varnode may be in the \e join space
   int4 overlap(const Address &op2loc,int4 op2size) const;	///< Return relative point of overlap with Address range
   uintb getNZMask(void) const { return nzm; } ///< Get the mask of bits within \b this that are known to be zero
   int4 termOrder(const Varnode *op) const; ///< Compare two Varnodes based on their term order

Ghidra/Features/Decompiler/src/decompile/datatests/retstruct.xml

@@ -0,0 +1,27 @@
+<decompilertest>
+<binaryimage arch="x86:LE:32:default:gcc">
+<!--
+    Example function returning a structure stored across multiple registers
+-->
+<bytechunk space="ram" offset="0x100000">
+5589e5538b45088b550c837d10007f05
+6bc064eb088d4007ba1e0000008d65fc
+5bc9c3
+</bytechunk>
+<symbol space="ram" offset="0x100000" name="retstruct"/>
+</binaryimage>
+<script>
+  <com>parse line struct foo { int4 a; int4 b; };</com>
+  <com>parse line extern foo retstruct(int4 x,int4 y,int4 z);</com>
+  <com>lo fu retstruct</com>
+  <com>type varnode %EAX(0x100010) int4 tmp</com>
+  <com>decompile</com>
+  <com>print C</com>
+</script>
+<stringmatch name="Return Structure #1" min="1" max="1">tmp = x \* 100;</stringmatch>
+<stringmatch name="Return Structure #2" min="1" max="1">tmp = x \+ 7;</stringmatch>
+<stringmatch name="Return Structure #3" min="1" max="1">y = 0x1e;</stringmatch>
+<stringmatch name="Return Structure #4" min="1" max="1">fVar1\.a = tmp;</stringmatch>
+<stringmatch name="Return Structure #5" min="1" max="1">fVar1\.b = y;</stringmatch>
+<stringmatch name="Return Structure #6" min="1" max="1">return fVar1;</stringmatch>
+</decompilertest>

Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileProcess.java

@@ -377,6 +377,9 @@ public class DecompileProcess {
 					currentResponse = null;		// Reset current buffer as a native message may follow
 					break;
 				case 16:			// Beginning of any native message from the decompiler
+					if (currentResponse != null) {	// Beginning of native message before end of main response
+						currentResponse.clear();	// Don't try to parse main response
+					}
 					currentResponse = stringDecoder;
 					currentResponse.open(1 << 20, programSource);
 					break;