Adjustments to default hidden return assignment action

Ghidra/Features/Decompiler/src/decompile/cpp/fspec.cc

@@ -731,7 +731,8 @@ void ParamListStandard::assignMap(const PrototypePieces &proto,TypeFactory &type
   for(int4 i=0;i<proto.intypes.size();++i) {
     res.emplace_back();
     Datatype *dt = proto.intypes[i];
-    if (assignAddress(dt,proto,i,typefactory,status,res.back()) == AssignAction::fail)
+    uint4 responseCode = assignAddress(dt,proto,i,typefactory,status,res.back());
+    if (responseCode == AssignAction::fail || responseCode == AssignAction::no_assignment)
       throw ParamUnassignedError("Cannot assign parameter address for " + dt->getName());
   }
 }
@@ -1503,7 +1504,12 @@ void ParamListStandardOut::assignMap(const PrototypePieces &proto,TypeFactory &t
     return;			// Leave the address as invalid
   }
   uint4 responseCode = assignAddress(proto.outtype,proto,-1,typefactory,status,res.back());
-  if (responseCode != AssignAction::success) { // Could not assign an address (too big)
+
+  if (responseCode == AssignAction::fail)
+    responseCode = AssignAction::hiddenret_ptrparam;	// Invoke default hidden return input assignment action
+
+  if (responseCode == AssignAction::hiddenret_ptrparam || responseCode == AssignAction::hiddenret_specialreg ||
+      responseCode == AssignAction::hiddenret_specialreg_void) { // Could not assign an address (too big)
     AddrSpace *spc = spacebase;
     if (spc == (AddrSpace *)0)
       spc = typefactory.getArch()->getDefaultDataSpace();

Ghidra/Features/Decompiler/src/decompile/cpp/modelrules.cc

@@ -356,7 +356,7 @@ AssignAction *AssignAction::decodeAction(Decoder &decoder,const ParamListStandar
     action = new ConvertToPointer(res);
   }
   else if (elemId == ELEM_HIDDEN_RETURN) {
-    action = new HiddenReturnAssign(res,false);
+    action = new HiddenReturnAssign(res,hiddenret_specialreg);
   }
   else if (elemId == ELEM_JOIN_PER_PRIMITIVE) {
     bool consumeMostSig = false;
@@ -721,10 +721,10 @@ void ConsumeAs::decode(Decoder &decoder)
   decoder.closeElement(elemId);
 }
 
-HiddenReturnAssign::HiddenReturnAssign(const ParamListStandard *res,bool voidLock)
+HiddenReturnAssign::HiddenReturnAssign(const ParamListStandard *res,uint4 code)
   : AssignAction(res)
 {
-  retCode = voidLock ? hiddenret_specialreg_void : hiddenret_specialreg;
+  retCode = code;
 }
 
 uint4 HiddenReturnAssign::assignAddress(Datatype *dt,const PrototypePieces &proto,int4 pos,TypeFactory &tlist,
@@ -736,12 +736,24 @@ uint4 HiddenReturnAssign::assignAddress(Datatype *dt,const PrototypePieces &prot
 void HiddenReturnAssign::decode(Decoder &decoder)
 
 {
+  retCode = hiddenret_specialreg;
   uint4 elemId = decoder.openElement(ELEM_HIDDEN_RETURN);
+  for(;;) {
     uint4 attribId = decoder.getNextAttributeId();
     if (attribId == ATTRIB_VOIDLOCK)
       retCode = hiddenret_specialreg_void;
-  else
+    else if (attribId == ATTRIB_STRATEGY) {
+      string strategyString = decoder.readString();
+      if (strategyString == "normalparam")
+	retCode = hiddenret_ptrparam;
+      else if (strategyString == "special")
 	retCode = hiddenret_specialreg;
+      else
+	throw DecoderError("Bad <hidden_return> strategy: " + strategyString);
+    }
+    else
+      break;
+  }
   decoder.closeElement(elemId);
 }
 

Ghidra/Features/Decompiler/src/decompile/cpp/modelrules.hh

@@ -200,7 +200,8 @@ class AssignAction {
 public:
   enum {
     success,			///< Data-type is fully assigned
-    fail,			///< Action could not be applied (not enough resources)
+    fail,			///< Action could not be applied
+    no_assignment,		///< Do not assign storage for this parameter
     hiddenret_ptrparam,		///< Hidden return pointer as first input parameter
     hiddenret_specialreg,	///< Hidden return pointer in dedicated input register
     hiddenret_specialreg_void	///< Hidden return pointer, but no normal return
@@ -327,19 +328,24 @@ public:
   virtual void decode(Decoder &decoder);
 };
 
-/// \brief Allocate the return value as special input register
+/// \brief Allocate the return value as an input parameter
 ///
-/// The assignAddress() method signals with \b hiddenret_specialreg, indicating that the
+/// A pointer to where the return value is to be stored is passed in as an input parameter.
-/// input register assignMap() method should use storage class TYPECLASS_HIDDENRET to assign
+/// This action signals this by returning one of
-/// an additional input register to hold a pointer to the return value.  This is different than
+///   - \b hiddenret_ptrparam         - indicating the pointer is allocated as a normal input parameter
-/// the default \e hiddenret action that assigns a location based TYPECLASS_PTR and generally
+///   - \b hiddenret_specialreg       - indicating the pointer is passed in a dedicated register
-/// consumes a general purpose input register.
+///   - \b hiddenret_specialreg_void
+///
+/// Usually, if a hidden return input is present, the normal register used for return
+/// will also hold the pointer at the point(s) where the function returns.  A signal of
+/// \b hiddenret_specialreg_void indicates the normal return register is not used to pass back
+/// the pointer.
 class HiddenReturnAssign : public AssignAction {
   uint4 retCode;		///< The specific signal to pass back
 public:
-  HiddenReturnAssign(const ParamListStandard *res,bool voidLock);	///< Constructor
+  HiddenReturnAssign(const ParamListStandard *res,uint4 code);	///< Constructor
   virtual AssignAction *clone(const ParamListStandard *newResource) const {
-    return new HiddenReturnAssign(newResource, retCode == hiddenret_specialreg_void); }
+    return new HiddenReturnAssign(newResource, retCode); }
   virtual uint4 assignAddress(Datatype *dt,const PrototypePieces &proto,int4 pos,TypeFactory &tlist,
 			      vector<int4> &status,ParameterPieces &res) const;
   virtual void decode(Decoder &decoder);

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/ParamListStandard.java

@@ -131,7 +131,7 @@ public class ParamListStandard implements ParamList {
 
 	{
 		if (dt.isZeroLength()) {
-			return AssignAction.FAIL;
+			return AssignAction.NO_ASSIGNMENT;
 		}
 		for (ModelRule modelRule : modelRules) {
 			int responseCode = modelRule.assignAddress(dt, proto, pos, dtManager, status, res);
@@ -183,7 +183,7 @@ public class ParamListStandard implements ParamList {
 			ParameterPieces store = new ParameterPieces();
 			res.add(store);
 			int resCode = assignAddress(proto.intypes.get(i), proto, i, dtManager, status, store);
-			if (resCode == AssignAction.FAIL) {
+			if (resCode == AssignAction.FAIL || resCode == AssignAction.NO_ASSIGNMENT) {
 				// Do not continue to assign after first failure
 				++i;
 				while (i < proto.intypes.size()) {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/ParamListStandardOut.java

@@ -50,7 +50,13 @@ public class ParamListStandardOut extends ParamListStandard {
 			return;		// Don't assign storage for VOID
 		}
 		int responseCode = assignAddress(proto.outtype, proto, -1, dtManager, status, store);
-		if (responseCode != AssignAction.SUCCESS) {
+		if (responseCode == AssignAction.FAIL) {
+			// Invoke default hidden return input assignment action
+			responseCode = AssignAction.HIDDENRET_PTRPARAM;
+		}
+		if (responseCode == AssignAction.HIDDENRET_PTRPARAM ||
+			responseCode == AssignAction.HIDDENRET_SPECIALREG ||
+			responseCode == AssignAction.HIDDENRET_SPECIALREG_VOID) {
 			// If the storage is not assigned (because the datatype is too big) create a hidden input parameter
 			int sz = (spacebase == null) ? -1 : spacebase.getPointerSize();
 			DataType pointerType = dtManager.getPointer(proto.outtype, sz);

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/protorules/AssignAction.java

@@ -35,10 +35,11 @@ import ghidra.xml.*;
  */
 public abstract class AssignAction {
 	public static final int SUCCESS = 0;			// Data-type is fully assigned
-	public static final int FAIL = 1;				// Action could not be applied (not enough resources)
+	public static final int FAIL = 1;				// Action could not be applied
-	public static final int HIDDENRET_PTRPARAM = 2;	// Hidden return pointer as first input parameter
+	public static final int NO_ASSIGNMENT = 2;		// Do not assign a storage location
-	public static final int HIDDENRET_SPECIALREG = 3;	// Hidden return pointer in special register
+	public static final int HIDDENRET_PTRPARAM = 3;	// Hidden return pointer as first input parameter
-	public static final int HIDDENRET_SPECIALREG_VOID = 4;	// Hidden return pointer, but no normal return
+	public static final int HIDDENRET_SPECIALREG = 4;	// Hidden return pointer in special register
+	public static final int HIDDENRET_SPECIALREG_VOID = 5;	// Hidden return pointer, but no normal return
 
 	protected ParamListStandard resource;			// Resources to which this action applies
 
@@ -120,7 +121,7 @@ public abstract class AssignAction {
 			action = new ConvertToPointer(res);
 		}
 		else if (nm.equals(ELEM_HIDDEN_RETURN.name())) {
-			action = new HiddenReturnAssign(res, false);
+			action = new HiddenReturnAssign(res, HIDDENRET_SPECIALREG);
 		}
 		else if (nm.equals(ELEM_JOIN_PER_PRIMITIVE.name())) {
 			boolean consumeMostSig = res.getEntry(0).isBigEndian();

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/protorules/HiddenReturnAssign.java

@@ -29,26 +29,34 @@ import ghidra.util.xml.SpecXmlUtils;
 import ghidra.xml.*;
 
 /**
- * Allocate the return value as special input register
+ * Allocate the return value as an input parameter
  * 
- * The assignAddress() method signals with hiddenret_specialreg, indicating that the
+ * A pointer to where the return value is to be stored is passed in as an input parameter.
- * input register assignMap() method should use storage class TYPECLASS_HIDDENRET to assign
+ * This action signals this by returning one of
- * an additional input register to hold a pointer to the return value.  This is different than
+ *   - HIDDENRET_PTRPARAM         - indicating the pointer is allocated as a normal input parameter
- * the default hiddenret action that assigns a location based TYPECLASS_PTR and generally
+ *   - HIDDENRET_SPECIALREG       - indicating the pointer is passed in a dedicated register
- * consumes a general purpose input register.
+ *   - HIDDENRET_SPECIALREG_VOID
+ * 
+ * Usually, if a hidden return input is present, the normal register used for return
+ * will also hold the pointer at the point(s) where the function returns.  A signal of
+ * HIDDENRET_SPECIALREG_VOID indicates the normal return register is not used to pass back
+ * the pointer.
  */
 public class HiddenReturnAssign extends AssignAction {
 
+	static public final String STRATEGY_SPECIAL = "special";	// Return pointer in special reg
+	static public final String STRATEGY_NORMAL = "normalparam";	// Return pointer as normal param
+
 	private int retCode;		// The specific signal to pass back
 
-	public HiddenReturnAssign(ParamListStandard res, boolean voidLock) {
+	public HiddenReturnAssign(ParamListStandard res, int code) {
 		super(res);
-		retCode = voidLock ? HIDDENRET_SPECIALREG_VOID : HIDDENRET_SPECIALREG;
+		retCode = code;
 	}
 
 	@Override
 	public AssignAction clone(ParamListStandard newResource) throws InvalidInputException {
-		return new HiddenReturnAssign(newResource, retCode == HIDDENRET_SPECIALREG_VOID);
+		return new HiddenReturnAssign(newResource, retCode);
 	}
 
 	@Override
@@ -69,7 +77,10 @@ public class HiddenReturnAssign extends AssignAction {
 	@Override
 	public void encode(Encoder encoder) throws IOException {
 		encoder.openElement(ELEM_HIDDEN_RETURN);
-		if (retCode == HIDDENRET_SPECIALREG_VOID) {
+		if (retCode == HIDDENRET_PTRPARAM) {
+			encoder.writeString(ATTRIB_STRATEGY, STRATEGY_NORMAL);
+		}
+		else if (retCode == HIDDENRET_SPECIALREG_VOID) {
 			encoder.writeBool(ATTRIB_VOIDLOCK, true);
 		}
 		encoder.closeElement(ELEM_HIDDEN_RETURN);
@@ -79,6 +90,18 @@ public class HiddenReturnAssign extends AssignAction {
 	public void restoreXml(XmlPullParser parser) throws XmlParseException {
 		retCode = HIDDENRET_SPECIALREG;
 		XmlElement elem = parser.start(ELEM_HIDDEN_RETURN.name());
+		String strategyString = elem.getAttribute(ATTRIB_STRATEGY.name());
+		if (strategyString != null) {
+			if (strategyString.equals(STRATEGY_NORMAL)) {
+				retCode = HIDDENRET_PTRPARAM;
+			}
+			else if (strategyString.equals(STRATEGY_SPECIAL)) {
+				retCode = HIDDENRET_SPECIALREG;
+			}
+			else {
+				throw new XmlParseException("Bad <hidden_return> strategy: " + strategyString);
+			}
+		}
 		String voidLockString = elem.getAttribute(ATTRIB_VOIDLOCK.name());
 		if (SpecXmlUtils.decodeBoolean(voidLockString)) {
 			retCode = HIDDENRET_SPECIALREG_VOID;