More adjustments to boolean propagation

Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc

@@ -4393,7 +4393,7 @@ void PropagationState::step(void)
     inslot = op->getSlot(vn);
     return;
   }
-  if (op == vn->getDef())
+  if (inslot == -1)
     op = (PcodeOp *)0;
   else
     op = vn->getDef();

Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc

@@ -291,6 +291,15 @@ bool SubvariableFlow::tryReturnPull(PcodeOp *op,ReplaceVarnode *rvn,int4 slot)
 	return false;
       if (inworklist)
 	worklist.push_back(rep);
+      else if (retvn->isConstant() && retop != op) {
+	// Trace won't revisit this RETURN, so we need to generate patch now
+	patchlist.push_back(PatchRecord());
+	patchlist.back().type = 2;
+	patchlist.back().pullop = retop;
+	patchlist.back().in1 = rep;
+	patchlist.back().slot = slot;
+	pullcount += 1;
+      }
     }
     returnsTraversed = true;
   }

Ghidra/Features/Decompiler/src/decompile/cpp/type.hh

@@ -115,6 +115,7 @@ public:
   virtual Datatype *clone(void) const=0;	///< Clone the data-type
   virtual void saveXml(ostream &s) const;	///< Serialize the data-type to XML
   int4 typeOrder(const Datatype &op) const { if (this==&op) return 0; return compare(op,10); }	///< Order this with -op- datatype
+  int4 typeOrderBool(const Datatype &op) const;	///< Order \b this with -op-, treating \e bool data-type as special
   void saveXmlBasic(ostream &s) const;	///< Save basic data-type properties
   void saveXmlRef(ostream &s) const;	///< Write an XML reference of \b this to stream
 };
@@ -447,4 +448,19 @@ public:
   void cacheCoreTypes(void);				///< Cache common types
 };
 
+/// Order data-types, with special handling of the \e bool data-type. Data-types are compared
+/// using the normal ordering, but \e bool is ordered after all other data-types. A return value
+/// of 0 indicates the data-types are the same, -1 indicates that \b this is prefered (ordered earlier),
+/// and 1 indicates \b this is ordered later.
+/// \param op is the other data-type to compare with \b this
+/// \return -1, 0, or 1
+inline int4 Datatype::typeOrderBool(const Datatype &op) const
+
+{
+  if (this == &op) return 0;
+  if (metatype == TYPE_BOOL) return 1;		// Never prefer bool over other data-types
+  if (op.metatype == TYPE_BOOL) return -1;
+  return compare(op,10);
+}
+
 #endif

Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc

@@ -112,7 +112,12 @@ void HighVariable::updateFlags(void) const
   highflags &= ~flagsdirty; // Clear the dirty flag
 }
 
-/// Using Datatype::typeOrder, find the member Varnode with the most specific data-type.
+/// Find the member Varnode with the most \e specialized data-type, handling \e bool specially.
+/// Boolean data-types are \e specialized in the data-type lattice, but not all byte values are boolean values.
+/// Within the Varnode/PcodeOp tree, the \e bool data-type can only propagate to a Varnode if it is verified to
+/// only take the boolean values 0 and 1. Since the data-type representative represents the type of all
+/// instances, if any instance is not boolean, then the HighVariable cannot be boolean, even though \e bool
+/// is more specialized. This method uses Datatype::typeOrderBool() to implement the special handling.
 /// \return the representative member
 Varnode *HighVariable::getTypeRepresentative(void) const
 
@@ -129,7 +134,7 @@ Varnode *HighVariable::getTypeRepresentative(void) const
       if (vn->isTypeLock())
 	rep = vn;
     }
-    else if (0>vn->getType()->typeOrder(*rep->getType()))
+    else if (0>vn->getType()->typeOrderBool(*rep->getType()))
       rep = vn;
   }
   return rep;