New consume logic for return value

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc

@@ -3369,19 +3369,39 @@ void ActionDeadCode::markConsumedParameters(FuncCallSpecs *fc,vector<Varnode *>
   }
   for(int4 i=1;i<callOp->numInput();++i) {
     Varnode *vn = callOp->getIn(i);
-    uintb consumeVal = vn->getNZMask();
-    if (vn->isAutoLive() || consumeVal > 0xffffffff)
+    uintb consumeVal;
+    if (vn->isAutoLive())
       consumeVal = ~((uintb)0);
-    else if (consumeVal > 0xffff)
-      consumeVal = 0xffffffff;
-    else if (consumeVal > 0xff)
-      consumeVal = 0xffff;
     else
-      consumeVal = 0xff;
+      consumeVal = minimalMask(vn->getNZMask());
     pushConsumed(consumeVal,vn,worklist);
   }
 }
 
+/// \brief Determine how the \e return \e values for the given function are consumed
+///
+/// Examine each CPUI_RETURN to see how the Varnode input is consumed.
+/// If the function's prototype is locked, assume the Varnode is entirely consumed.
+/// If there are no CPUI_RETURN ops, return 0
+/// \param data is the given function
+/// \return the bit mask of what is consumed
+uintb ActionDeadCode::gatherConsumedReturn(Funcdata &data)
+
+{
+  list<PcodeOp *>::const_iterator iter,enditer;
+  enditer = data.endOp(CPUI_RETURN);
+  uintb consumeVal = 0;
+  for(iter=data.beginOp(CPUI_RETURN);iter!=enditer;++iter) {
+    PcodeOp *returnOp = *iter;
+    if (returnOp->isDead()) continue;
+    if (returnOp->numInput() > 1) {
+      Varnode *vn = returnOp->getIn(1);
+      consumeVal |= minimalMask(vn->getNZMask());
+    }
+  }
+  return consumeVal;
+}
+
 int4 ActionDeadCode::apply(Funcdata &data)
 
 {
@@ -3389,6 +3409,7 @@ int4 ActionDeadCode::apply(Funcdata &data)
   list<PcodeOp *>::const_iterator iter;
   PcodeOp *op;
   Varnode *vn;
+  uintb returnConsume;
   vector<Varnode *> worklist;
   VarnodeLocSet::const_iterator viter,endviter;
   const AddrSpaceManager *manage = data.getArch();
@@ -3417,6 +3438,7 @@ int4 ActionDeadCode::apply(Funcdata &data)
     }
   }
 
+  returnConsume = gatherConsumedReturn(data);
   for(iter=data.beginOpAlive();iter!=data.endOpAlive();++iter) {
     op = *iter;
 
@@ -3427,10 +3449,21 @@ int4 ActionDeadCode::apply(Funcdata &data)
 	  pushConsumed(~((uintb)0),op->getIn(i),worklist);
       }
       // Postpone setting consumption on CALL and CALLIND inputs
+      if (!op->isAssignment())
+	continue;
     }
     else if (!op->isAssignment()) {
-      for(i=0;i<op->numInput();++i)
-	pushConsumed(~((uintb)0),op->getIn(i),worklist);
+      if (op->code() == CPUI_RETURN) {
+	pushConsumed(~((uintb)0),op->getIn(0),worklist);
+	if (op->numInput() > 1)
+	  pushConsumed(returnConsume,op->getIn(1),worklist);
+      }
+      else {
+	for(i=0;i<op->numInput();++i)
+	  pushConsumed(~((uintb)0),op->getIn(i),worklist);
+      }
+      // Postpone setting consumption on RETURN input
+      continue;
     }
     else {
       for(i=0;i<op->numInput();++i) {
@@ -3440,7 +3473,7 @@ int4 ActionDeadCode::apply(Funcdata &data)
       }
     }
     vn = op->getOut();
-    if ((vn!=(Varnode *)0)&&(vn->isAutoLive()))
+    if (vn->isAutoLive())
       pushConsumed(~((uintb)0),vn,worklist);
   }
 

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.hh

@@ -542,6 +542,8 @@ class ActionDeadCode : public Action {
   static void propagateConsumed(vector<Varnode *> &worklist);
   static bool neverConsumed(Varnode *vn,Funcdata &data);
   static void markConsumedParameters(FuncCallSpecs *fc,vector<Varnode *> &worklist);
+  static uintb gatherConsumedReturn(Funcdata &data);
+  static uintb minimalMask(uintb val);		///< Calculate smallest mask that covers the given value
 public:
   ActionDeadCode(const string &g) : Action(0,"deadcode",g) {}	///< Constructor
   virtual Action *clone(const ActionGroupList &grouplist) const {
@@ -1073,4 +1075,20 @@ public:
 inline bool TermOrder::additiveCompare(const PcodeOpEdge *op1,const PcodeOpEdge *op2) {
     return (-1 == op1->getVarnode()->termOrder(op2->getVarnode())); }
 
+/// Calculcate a mask that covers either the least significant byte, uint2, uint4, or uint8,
+/// whatever is smallest.
+/// \param val is the given value
+/// \return the minimal mask
+inline uintb ActionDeadCode::minimalMask(uintb val)
+
+{
+  if (val > 0xffffffff)
+    return ~((uintb)0);
+  if (val > 0xffff)
+    return 0xffffffff;
+  if (val > 0xff)
+    return 0xffff;
+  return 0xff;
+}
+
 #endif

Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc

@@ -269,6 +269,10 @@ bool SubvariableFlow::tryReturnPull(PcodeOp *op,ReplaceVarnode *rvn,int4 slot)
 {
   if (slot == 0) return false;	// Don't deal with actual return address container
   if (fd->getFuncProto().isOutputLocked()) return false;
+  if (!aggressive) {
+    if ((rvn->vn->getConsume()&~rvn->mask)!=0)	// If there's something outside the mask being consumed
+      return false;				// Don't truncate
+  }
 
   if (!returnsTraversed) {
     // If we plan to truncate the size of a return variable, we need to propagate the logical size to any other
@@ -557,7 +561,6 @@ bool SubvariableFlow::traceForward(ReplaceVarnode *rvn)
       hcount += 1;		// Dealt with this descendant
       break;
     case CPUI_RETURN:
-      if (!aggressive) return false;
       if (!tryReturnPull(op,rvn,slot)) return false;
       hcount += 1;
       break;
@@ -835,7 +838,6 @@ bool SubvariableFlow::traceForwardSext(ReplaceVarnode *rvn)
       hcount += 1;		// Dealt with this descendant
       break;
     case CPUI_RETURN:
-      if (!aggressive) return false;
       if (!tryReturnPull(op,rvn,slot)) return false;
       hcount += 1;
       break;