From e6f09b141b47d83e386bb2d7c78fa38f5159a130 Mon Sep 17 00:00:00 2001
From: caheckman <48068198+caheckman@users.noreply.github.com>
Date: Tue, 11 Feb 2020 13:02:53 -0500
Subject: [PATCH] More adjustments to boolean propagation

---
 .../Decompiler/src/decompile/cpp/coreaction.cc   |  2 +-
 .../Decompiler/src/decompile/cpp/subflow.cc      |  9 +++++++++
 .../Decompiler/src/decompile/cpp/type.hh         | 16 ++++++++++++++++
 .../Decompiler/src/decompile/cpp/variable.cc     |  9 +++++++--
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc b/Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc
index e337d8fc4c..cedc769720 100644
--- a/Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc
+++ b/Ghidra/Features/Decompiler/src/decompile/cpp/coreaction.cc
@@ -4393,7 +4393,7 @@ void PropagationState::step(void)
     inslot = op->getSlot(vn);
     return;
   }
-  if (op == vn->getDef())
+  if (inslot == -1)
     op = (PcodeOp *)0;
   else
     op = vn->getDef();
diff --git a/Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc b/Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc
index 76009c6c87..0ebcb283be 100644
--- a/Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc
+++ b/Ghidra/Features/Decompiler/src/decompile/cpp/subflow.cc
@@ -291,6 +291,15 @@ bool SubvariableFlow::tryReturnPull(PcodeOp *op,ReplaceVarnode *rvn,int4 slot)
 	return false;
       if (inworklist)
 	worklist.push_back(rep);
+      else if (retvn->isConstant() && retop != op) {
+	// Trace won't revisit this RETURN, so we need to generate patch now
+	patchlist.push_back(PatchRecord());
+	patchlist.back().type = 2;
+	patchlist.back().pullop = retop;
+	patchlist.back().in1 = rep;
+	patchlist.back().slot = slot;
+	pullcount += 1;
+      }
     }
     returnsTraversed = true;
   }
diff --git a/Ghidra/Features/Decompiler/src/decompile/cpp/type.hh b/Ghidra/Features/Decompiler/src/decompile/cpp/type.hh
index aaff1c0491..ab60be1820 100644
--- a/Ghidra/Features/Decompiler/src/decompile/cpp/type.hh
+++ b/Ghidra/Features/Decompiler/src/decompile/cpp/type.hh
@@ -115,6 +115,7 @@ public:
   virtual Datatype *clone(void) const=0;	///< Clone the data-type
   virtual void saveXml(ostream &s) const;	///< Serialize the data-type to XML
   int4 typeOrder(const Datatype &op) const { if (this==&op) return 0; return compare(op,10); }	///< Order this with -op- datatype
+  int4 typeOrderBool(const Datatype &op) const;	///< Order \b this with -op-, treating \e bool data-type as special
   void saveXmlBasic(ostream &s) const;	///< Save basic data-type properties
   void saveXmlRef(ostream &s) const;	///< Write an XML reference of \b this to stream
 };
@@ -447,4 +448,19 @@ public:
   void cacheCoreTypes(void);				///< Cache common types
 };
 
+/// Order data-types, with special handling of the \e bool data-type. Data-types are compared
+/// using the normal ordering, but \e bool is ordered after all other data-types. A return value
+/// of 0 indicates the data-types are the same, -1 indicates that \b this is prefered (ordered earlier),
+/// and 1 indicates \b this is ordered later.
+/// \param op is the other data-type to compare with \b this
+/// \return -1, 0, or 1
+inline int4 Datatype::typeOrderBool(const Datatype &op) const
+
+{
+  if (this == &op) return 0;
+  if (metatype == TYPE_BOOL) return 1;		// Never prefer bool over other data-types
+  if (op.metatype == TYPE_BOOL) return -1;
+  return compare(op,10);
+}
+
 #endif
diff --git a/Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc b/Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc
index 2c8d6cd14a..ef4da743b9 100644
--- a/Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc
+++ b/Ghidra/Features/Decompiler/src/decompile/cpp/variable.cc
@@ -112,7 +112,12 @@ void HighVariable::updateFlags(void) const
   highflags &= ~flagsdirty; // Clear the dirty flag
 }
 
-/// Using Datatype::typeOrder, find the member Varnode with the most specific data-type.
+/// Find the member Varnode with the most \e specialized data-type, handling \e bool specially.
+/// Boolean data-types are \e specialized in the data-type lattice, but not all byte values are boolean values.
+/// Within the Varnode/PcodeOp tree, the \e bool data-type can only propagate to a Varnode if it is verified to
+/// only take the boolean values 0 and 1. Since the data-type representative represents the type of all
+/// instances, if any instance is not boolean, then the HighVariable cannot be boolean, even though \e bool
+/// is more specialized. This method uses Datatype::typeOrderBool() to implement the special handling.
 /// \return the representative member
 Varnode *HighVariable::getTypeRepresentative(void) const
 
@@ -129,7 +134,7 @@ Varnode *HighVariable::getTypeRepresentative(void) const
       if (vn->isTypeLock())
 	rep = vn;
     }
-    else if (0>vn->getType()->typeOrder(*rep->getType()))
+    else if (0>vn->getType()->typeOrderBool(*rep->getType()))
       rep = vn;
   }
   return rep;