downcast capability in LaneDivide, moved LaneDivide to stackstall

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc

@@ -504,7 +504,8 @@ int4 ActionStackPtrFlow::apply(Funcdata &data)
 /// \param data is the function being transformed
 /// \param vn is the given single Varnode
 /// \param lanedRegister is acceptable set of lane sizes for the Varnode
-bool ActionLaneDivide::processVarnode(Funcdata &data,Varnode *vn,const LanedRegister &lanedRegister)
+/// \param allowDowncast is \b true if we allow lane systems with SUBPIECE terminators
+bool ActionLaneDivide::processVarnode(Funcdata &data,Varnode *vn,const LanedRegister &lanedRegister,bool allowDowncast)
 
 {
   list<PcodeOp *>::const_iterator iter = vn->beginDescend();
@@ -524,7 +525,7 @@ bool ActionLaneDivide::processVarnode(Funcdata &data,Varnode *vn,const LanedRegi
       }
       if (!description.subset(bytePos,vn->getSize()))		// Try to restrict lane scheme to actual Varnode
 	continue;
-      LaneDivide laneDivide(&data,op->getIn(0),description);
+      LaneDivide laneDivide(&data,op->getIn(0),description,allowDowncast);
       if (laneDivide.doTrace()) {
 	laneDivide.apply();
 	count += 1;		// Indicate a change was made
@@ -537,16 +538,23 @@ bool ActionLaneDivide::processVarnode(Funcdata &data,Varnode *vn,const LanedRegi
 
 /// \brief Search for and attempt to split Varnodes that match the given laned vector register
 ///
+/// All varnodes (bigger than 8 bytes) that are contained in the vector register are processed,
+/// looking for a working lane scheme, and splitting based on that scheme. Return \b false if there
+/// is at least one eligible Varnode that could not be split.
 /// \param data is the function being modified
 /// \param lanedRegister is the given register and acceptable lane schemes
+/// \param allowDowncast is \b true if SUBPIECE terminators are allowed in the schemes
 /// \param iter is an iterator to the first Varnode that matches the vector register
-void ActionLaneDivide::processLane(Funcdata &data,const LanedRegister &lanedRegister,VarnodeLocSet::const_iterator iter)
+/// \return \b true if all varnodes were successfully split
+bool ActionLaneDivide::processLane(Funcdata &data,const LanedRegister &lanedRegister,bool allowDowncast,
+				   VarnodeLocSet::const_iterator iter)
 
 {
   int4 fullSize = lanedRegister.getStorage().size;
   Address startAddress(lanedRegister.getStorage().getAddr());
   Address lastAddress(startAddress + (fullSize-1));
   VarnodeLocSet::const_iterator enditer = data.endLoc();
+  bool res = true;
   while(iter != enditer) {
     Varnode *vn = *iter;
     ++iter;
@@ -557,11 +565,15 @@ void ActionLaneDivide::processLane(Funcdata &data,const LanedRegister &lanedRegi
     int4 diff = (int4)(vn->getOffset() - startAddress.getOffset());
     if (diff + vn->getSize() > fullSize)	// Must be contained by full register
       continue;
-    if (processVarnode(data,vn,lanedRegister)) {
+    if (processVarnode(data,vn,lanedRegister,allowDowncast)) {
       // If changes were made, iterator may no longer be valid, generate a new one
       iter = data.beginLoc(startAddress);
+      res = true;	// We may have eliminated a previous failure
     }
+    else
+      res = false;	// Note the failure
   }
+  return res;
 }
 
 int4 ActionLaneDivide::apply(Funcdata &data)
@@ -578,7 +590,8 @@ int4 ActionLaneDivide::apply(Funcdata &data)
     if (viter == data.endLoc()) break;
     Varnode *vn = *viter;
     if (lastAddress < vn->getAddr()) break;
-    processLane(data,lanedRegister,viter);
+    if (!processLane(data,lanedRegister,false,viter))	// Try without SUBPIECE terminators
+      processLane(data,lanedRegister,true,viter);	// If we fail, try with SUBPIECE terminators
   }
   return 0;
 }
@@ -4594,7 +4607,6 @@ void universal_action(Architecture *conf)
       //      actmainloop->addAction( new ActionParamShiftStop("paramshift") );
       actmainloop->addAction( new ActionRestrictLocal("localrecovery") ); // Do before dead code removed
       actmainloop->addAction( new ActionDeadCode("deadcode") );
-      actmainloop->addAction( new ActionLaneDivide("base") );
       actmainloop->addAction( new ActionDynamicMapping("dynamic") ); // Must come before restructurevarnode and infertypes
       actmainloop->addAction( new ActionRestructureVarnode("localrecovery") );
       actmainloop->addAction( new ActionSpacebase("base") );	// Must come before infertypes and nonzeromask
@@ -4725,6 +4737,7 @@ void universal_action(Architecture *conf)
 	conf->extra_pool_rules.clear(); // Rules are now absorbed into universal
       }
       actstackstall->addAction( actprop );
+      actstackstall->addAction( new ActionLaneDivide("base") );
       actstackstall->addAction( new ActionMultiCse("analysis") );
       actstackstall->addAction( new ActionShadowVar("analysis") );
       actstackstall->addAction( new ActionDeindirect("deindirect") );

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.hh

@@ -104,8 +104,8 @@ public:
 /// if a particular lane scheme makes sense in terms of the function's data-flow, and then
 /// rewrites the data-flow so that the lanes become explicit Varnodes.
 class ActionLaneDivide : public Action {
-  bool processVarnode(Funcdata &data,Varnode *vn,const LanedRegister &lanedRegister);
+  bool processVarnode(Funcdata &data,Varnode *vn,const LanedRegister &lanedRegister,bool allowDowncast);
-  void processLane(Funcdata &data,const LanedRegister &lanedRegister,VarnodeLocSet::const_iterator iter);
+  bool processLane(Funcdata &data,const LanedRegister &lanedRegister,bool allowDowncast,VarnodeLocSet::const_iterator iter);
 public:
   ActionLaneDivide(const string &g) : Action(rule_onceperfunc,"lanedivide",g) {}	///< Constructor
   virtual Action *clone(const ActionGroupList &grouplist) const {

Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc

@@ -2237,8 +2237,21 @@ bool LaneDivide::traceForward(TransformVar *rvn,int4 numLanes,int4 skipLanes)
       {
 	int4 bytePos = (int4)op->getIn(1)->getOffset();
 	int4 outLanes,outSkip;
-	if (!description.restriction(numLanes, skipLanes, bytePos, outvn->getSize(), outLanes, outSkip))
+	if (!description.restriction(numLanes, skipLanes, bytePos, outvn->getSize(), outLanes, outSkip)) {
+	  if (allowSubpieceTerminator) {
+	    int4 laneIndex = description.getBoundary(bytePos);
+	    if (laneIndex < 0 || laneIndex >= description.getNumLanes())	// Does piece start on lane boundary?
+	      return false;
+	    if (description.getSize(laneIndex) <= outvn->getSize())		// Is the piece smaller than a lane?
+	      return false;
+	    // Treat SUBPIECE as terminating
+	    TransformOp *rop = newPreexistingOp(2, CPUI_SUBPIECE, op);
+	    opSetInput(rop, rvn + (laneIndex - skipLanes), 0);
+	    opSetInput(rop, newConstant(4, 0, 0), 1);
+	    break;
+	  }
 	  return false;
+	}
 	if (outLanes == 1) {
 	  TransformOp *rop = newPreexistingOp(1, CPUI_COPY, op);
 	  opSetInput(rop,rvn + (outSkip-skipLanes), 0);
@@ -2379,10 +2392,12 @@ bool LaneDivide::processNextWork(void)
 /// \param f is the function being transformed
 /// \param root is the root Varnode to start tracing lanes from
 /// \param desc is a description of the lanes on the root Varnode
-LaneDivide::LaneDivide(Funcdata *f,Varnode *root,const LaneDescription &desc)
+/// \param allowDowncast is \b true if we all SUBPIECE to be treated as terminating
+LaneDivide::LaneDivide(Funcdata *f,Varnode *root,const LaneDescription &desc,bool allowDowncast)
   : TransformManager(f), description(desc)
 {
-   setReplacement(root, desc.getNumLanes(), 0);
+  allowSubpieceTerminator = allowDowncast;
+  setReplacement(root, desc.getNumLanes(), 0);
 }
 
 /// Push the lanes around from the root, setting up the explicit transforms as we go.

Ghidra/Features/Decompiler/src/decompile/cpp/subflow.hh

@@ -165,6 +165,7 @@ class LaneDivide : public TransformManager {
 
   LaneDescription description;	///< Global description of lanes that need to be split
   vector<WorkNode> workList;	///< List of Varnodes still left to trace
+  bool allowSubpieceTerminator;	///< \b true if we allow lanes to be cast (via SUBPIECE) to a smaller integer size
 
   TransformVar *setReplacement(Varnode *vn,int4 numLanes,int4 skipLanes);
   void buildUnaryOp(OpCode opc,PcodeOp *op,TransformVar *inVars,TransformVar *outVars,int4 numLanes);
@@ -178,7 +179,7 @@ class LaneDivide : public TransformManager {
   bool traceBackward(TransformVar *rvn,int4 numLanes,int4 skipLanes);
   bool processNextWork(void);		///< Process the next Varnode on the work list
 public:
-  LaneDivide(Funcdata *f,Varnode *root,const LaneDescription &desc);	///< Constructor
+  LaneDivide(Funcdata *f,Varnode *root,const LaneDescription &desc,bool allowDowncast);	///< Constructor
   bool doTrace(void);		///< Trace lanes as far as possible from the root Varnode
 };