GP-4688 Tighter checks on NaN expression truncation

This commit is contained in:
caheckman 2024-06-14 00:55:25 +00:00
parent 0a3f1bad3f
commit b650848848
2 changed files with 44 additions and 24 deletions

View file

@ -9476,6 +9476,26 @@ bool RuleIgnoreNan::checkBackForCompare(Varnode *floatVar,Varnode *root)
return false; return false;
} }
/// \brief Test if the given Varnode is produced by a NaN operation.
///
/// The Varnode can be the direct or negated output of a NaN.
/// \param vn is the given Varnode
/// \return \b true if the Varnode is the output of the NaN
bool RuleIgnoreNan::isAnotherNan(Varnode *vn)
{
if (!vn->isWritten()) return false;
PcodeOp *op = vn->getDef();
OpCode opc = op->code();
if (opc == CPUI_BOOL_NEGATE) {
vn = op->getIn(0);
if (!vn->isWritten()) return false;
op = vn->getDef();
opc = op->code();
}
return (opc == CPUI_FLOAT_NAN);
}
/// \brief Test if a boolean expression incorporates a floating-point comparison, and remove the NaN data-flow if it does /// \brief Test if a boolean expression incorporates a floating-point comparison, and remove the NaN data-flow if it does
/// ///
/// The given PcodeOp takes input from a NaN operation through a specific slot. We look for a floating-point comparison /// The given PcodeOp takes input from a NaN operation through a specific slot. We look for a floating-point comparison
@ -9497,36 +9517,35 @@ Varnode *RuleIgnoreNan::testForComparison(Varnode *floatVar,PcodeOp *op,int4 slo
{ {
if (op->code() == matchCode) { if (op->code() == matchCode) {
Varnode *vn = op->getIn(1-slot); Varnode *vn = op->getIn(1 - slot);
if (checkBackForCompare(floatVar,vn)) { if (checkBackForCompare(floatVar,vn)) {
data.opSetOpcode(op, CPUI_COPY); data.opSetOpcode(op,CPUI_COPY);
data.opRemoveInput(op, 1); data.opRemoveInput(op,1);
data.opSetInput(op, vn, 0); data.opSetInput(op,vn,0);
count += 1; count += 1;
} }
else if (isAnotherNan(vn)) {
return op->getOut(); return op->getOut();
} }
if (op->code() != CPUI_CBRANCH) }
return (Varnode *)0; else if (op->code() == CPUI_CBRANCH) {
BlockBasic *parent = op->getParent(); BlockBasic *parent = op->getParent();
bool flowToFromCompare = false;
PcodeOp *lastOp; PcodeOp *lastOp;
int4 outDir = (matchCode == CPUI_BOOL_OR) ? 0 : 1; int4 outDir = (matchCode == CPUI_BOOL_OR) ? 0 : 1;
if (op->isBooleanFlip()) if (op->isBooleanFlip())
outDir = 1 - outDir; outDir = 1 - outDir;
FlowBlock *outBranch = parent->getOut(outDir); FlowBlock *outBranch = parent->getOut(outDir);
lastOp = outBranch->lastOp(); lastOp = outBranch->lastOp();
if (lastOp != (PcodeOp *)0 && lastOp->code() == CPUI_CBRANCH) { if (lastOp != (PcodeOp*)0 && lastOp->code() == CPUI_CBRANCH) {
FlowBlock *otherBranch = parent->getOut(1-outDir); FlowBlock *otherBranch = parent->getOut(1 - outDir);
if (outBranch->getOut(0) == otherBranch || outBranch->getOut(1) == otherBranch) { if (outBranch->getOut(0) == otherBranch || outBranch->getOut(1) == otherBranch) {
if (checkBackForCompare(floatVar, lastOp->getIn(1))) if (checkBackForCompare(floatVar,lastOp->getIn(1))) {
flowToFromCompare = true; data.opSetInput(op,data.newConstant(1,(matchCode == CPUI_BOOL_OR) ? 0 : 1),1);
}
}
if (flowToFromCompare) {
data.opSetInput(op,data.newConstant(1, 0),1); // Treat result of NaN as false
count += 1; count += 1;
} }
}
}
}
return (Varnode *)0; return (Varnode *)0;
} }

View file

@ -1550,6 +1550,7 @@ public:
class RuleIgnoreNan : public Rule { class RuleIgnoreNan : public Rule {
static bool checkBackForCompare(Varnode *floatVar,Varnode *root); static bool checkBackForCompare(Varnode *floatVar,Varnode *root);
static bool isAnotherNan(Varnode *vn);
static Varnode *testForComparison(Varnode *floatVar,PcodeOp *op,int4 slot,OpCode matchCode,int4 &count,Funcdata &data); static Varnode *testForComparison(Varnode *floatVar,PcodeOp *op,int4 slot,OpCode matchCode,int4 &count,Funcdata &data);
public: public:
RuleIgnoreNan(const string &g) : Rule( g, 0, "ignorenan") {} ///< Constructor RuleIgnoreNan(const string &g) : Rule( g, 0, "ignorenan") {} ///< Constructor