New tests for data-types, test debugging infrastructure

Ghidra/Features/Decompiler/src/decompile/cpp/Makefile

@@ -94,7 +94,7 @@ GHIDRA=	ghidra_arch inject_ghidra ghidra_translate loadimage_ghidra \
 # Additional files specific to the sleigh compiler
 SLACOMP=slgh_compile slghparse slghscan
 # Additional special files that should not be considered part of the library
-SPECIAL=consolemain sleighexample test testfunction
+SPECIAL=consolemain sleighexample test
 # Any additional modules for the command line decompiler
 EXTRA= $(filter-out $(CORE) $(DECCORE) $(SLEIGH) $(GHIDRA) $(SLACOMP) $(SPECIAL),$(ALL_NAMES))
 
@@ -117,7 +117,7 @@ COMMANDLINE_NAMES=$(CORE) $(DECCORE) $(EXTRA) $(SLEIGH) consolemain
 COMMANDLINE_DEBUG=-DCPUI_DEBUG -D__TERMINAL__
 COMMANDLINE_OPT=-D__TERMINAL__
 
-TEST_NAMES=$(CORE) $(DECCORE) $(SLEIGH) $(EXTRA) testfunction test 
+TEST_NAMES=$(CORE) $(DECCORE) $(SLEIGH) $(EXTRA) test 
 TEST_DEBUG=-D__TERMINAL__
 
 GHIDRA_NAMES=$(CORE) $(DECCORE) $(GHIDRA)

Ghidra/Features/Decompiler/src/decompile/cpp/database.cc

@@ -1688,6 +1688,7 @@ Symbol *Scope::addConvertSymbol(uint4 format,uintb value,Address &addr,uint8 has
   Symbol *sym;
 
   sym = new EquateSymbol(owner,"",format,value);
+  addSymbolInternal(sym);
   RangeList rnglist;
   if (!addr.isInvalid())
     rnglist.insertRange(addr.getSpace(),addr.getOffset(),addr.getOffset());

Ghidra/Features/Decompiler/src/decompile/cpp/ifacedecomp.cc

@@ -130,6 +130,9 @@ void IfaceDecompCapability::registerCommands(IfaceStatus *status)
   status->registerCom(new IfcPreferSplit(),"prefersplit");
   status->registerCom(new IfcStructureBlocks(),"structure","blocks");
   status->registerCom(new IfcAnalyzeRange(), "analyze","range");
+  status->registerCom(new IfcLoadTestFile(), "load","test","file");
+  status->registerCom(new IfcListTestCommands(), "list","test","commands");
+  status->registerCom(new IfcExecuteTestCommand(), "execute","test","command");
 #ifdef CPUI_RULECOMPILE
   status->registerCom(new IfcParseRule(),"parse","rule");
   status->registerCom(new IfcExperimentalRules(),"experimental","rules");
@@ -218,6 +221,7 @@ IfaceDecompData::IfaceDecompData(void)
   conf = (Architecture *)0;
   fd = (Funcdata *)0;
   cgraph = (CallGraph *)0;
+  testCollection = (FunctionTestCollection *)0;
 #ifdef OPACTION_DEBUG
   jumptabledebug = false;
 #endif
@@ -230,6 +234,8 @@ IfaceDecompData::~IfaceDecompData(void)
     delete cgraph;
   if (conf != (Architecture *)0)
     delete conf;
+  if (testCollection != (FunctionTestCollection *)0)
+    delete testCollection;
 // fd will get deleted with Database
 }
 
@@ -3143,6 +3149,71 @@ void IfcAnalyzeRange::execute(istream &s)
   }
 }
 
+/// \class IfcLoadTestFile
+/// \brief Load a datatest environment file: `load test <filename>`
+///
+/// The program and associated script from a decompiler test file is loaded
+void IfcLoadTestFile::execute(istream &s)
+
+{
+  string filename;
+
+  if (dcp->conf != (Architecture *)0)
+    throw IfaceExecutionError("Load image already present");
+  s >> filename;
+  dcp->testCollection = new FunctionTestCollection(status);
+  dcp->testCollection->loadTest(filename);
+  *status->optr << filename << " test successfully loaded: " << dcp->conf->getDescription() << endl;
+}
+
+/// \class IfaceListTestCommands
+/// \brief List all the script commands in the current test: `list test commands`
+void IfcListTestCommands::execute(istream &s)
+
+{
+  if (dcp->testCollection == (FunctionTestCollection *)0)
+    throw IfaceExecutionError("No test file is loaded");
+  for(int4 i=0;i<dcp->testCollection->numCommands();++i) {
+    *status->optr << ' ' << dec << i+1 << ": " << dcp->testCollection->getCommand(i) << endl;
+  }
+}
+
+/// \class IfcExecuteTestCommands
+/// \brief Execute a specified range of the test script: `execute test command <#>-<#>
+void IfcExecuteTestCommand::execute(istream &s)
+
+{
+  if (dcp->testCollection == (FunctionTestCollection *)0)
+    throw IfaceExecutionError("No test file is loaded");
+  int4 first = -1;
+  int4 last = -1;
+  char hyphen;
+
+  s >> ws >> dec >> first;
+  first -= 1;
+  if (first < 0 || first > dcp->testCollection->numCommands())
+    throw IfaceExecutionError("Command index out of bounds");
+  s >> ws;
+  if (!s.eof()) {
+    s >> ws >> hyphen;
+    if (hyphen != '-')
+      throw IfaceExecutionError("Missing hyphenated command range");
+    s >> ws >> last;
+    last -= 1;
+    if (last < 0 || last < first || last > dcp->testCollection->numCommands())
+      throw IfaceExecutionError("Command index out of bounds");
+  }
+  else {
+    last = first;
+  }
+  ostringstream s1;
+  for(int4 i=first;i<=last;++i) {
+    s1 << dcp->testCollection->getCommand(i) << endl;
+  }
+  istringstream *s2 = new istringstream(s1.str());
+  status->pushScript(s2, "test> ");
+}
+
 #ifdef OPACTION_DEBUG
 
 void IfcDebugAction::execute(istream &s)

Ghidra/Features/Decompiler/src/decompile/cpp/ifacedecomp.hh

@@ -19,11 +19,11 @@
 #ifndef __IFACE_DECOMP__
 #define __IFACE_DECOMP__
 
-#include "ifaceterm.hh"
 #include "graph.hh"
 #include "grammar.hh"
 #include "callgraph.hh"
 #include "paramid.hh"
+#include "testfunction.hh"
 #ifdef CPUI_RULECOMPILE
 #include "rulecompile.hh"
 #endif
@@ -44,6 +44,7 @@ public:
   Funcdata *fd;		///< Current function active in the console
   Architecture *conf;	///< Current architecture/program active in the console
   CallGraph *cgraph;	///< Call-graph information for the program
+  FunctionTestCollection *testCollection;		///< Executable environment from a datatest
 
   map<Funcdata*,PrototypePieces> prototypePieces;
   void storePrototypePieces( Funcdata *fd_in, PrototypePieces pp_in ) { prototypePieces.insert(pair<Funcdata*,PrototypePieces>(fd_in,pp_in)); }
@@ -568,6 +569,21 @@ public:
   virtual void execute(istream &s);
 };
 
+class IfcLoadTestFile : public IfaceDecompCommand {
+public:
+  virtual void execute(istream &s);
+};
+
+class IfcListTestCommands : public IfaceDecompCommand {
+public:
+  virtual void execute(istream &s);
+};
+
+class IfcExecuteTestCommand : public IfaceDecompCommand {
+public:
+  virtual void execute(istream &s);
+};
+
 #ifdef CPUI_RULECOMPILE
 class IfcParseRule : public IfaceDecompCommand {
 public:

Ghidra/Features/Decompiler/src/decompile/cpp/ifaceterm.cc

@@ -237,15 +237,12 @@ void IfaceTerm::readLine(string &line)
   } while(val != '\n');
 }
 
-void IfaceTerm::pushScript(const string &filename,const string &newprompt)
+void IfaceTerm::pushScript(istream *iptr,const string &newprompt)
 
 {
-  ifstream *s = new ifstream(filename.c_str());
-  if (!*s)
-    throw IfaceParseError("Unable to open script file");
   inputstack.push_back(sptr);
-  sptr = s;
+  sptr = iptr;
-  IfaceStatus::pushScript(filename,newprompt);
+  IfaceStatus::pushScript(iptr,newprompt);
 }
 
 void IfaceTerm::popScript(void)
@@ -254,6 +251,7 @@ void IfaceTerm::popScript(void)
   delete sptr;
   sptr = inputstack.back();
   inputstack.pop_back();
+  IfaceStatus::popScript();
 }
 
 bool IfaceTerm::isStreamFinished(void) const

Ghidra/Features/Decompiler/src/decompile/cpp/ifaceterm.hh

@@ -44,7 +44,7 @@ class IfaceTerm : public IfaceStatus {
 public:
   IfaceTerm(const string &prmpt,istream &is,ostream &os);	///< Constructor
   virtual ~IfaceTerm(void);
-  virtual void pushScript(const string &filename,const string &newprompt);
+  virtual void pushScript(istream *iptr,const string &newprompt);
   virtual void popScript(void);
   virtual bool isStreamFinished(void) const;
 };

Ghidra/Features/Decompiler/src/decompile/cpp/interface.cc

@@ -134,14 +134,27 @@ IfaceStatus::IfaceStatus(const string &prmpt,ostream &os,int4 mxhist)
   curhistory = 0;
 }
 
-/// \brief Provide a new script file to execute, with an associated command prompt
+/// \brief Push a new file on the script stack
 ///
-/// The script provides a subsidiary input stream to the current stream.
+/// Attempt to open the file, and if we succeed put the open stream onto the script stack.
-/// Once commands from the script are complete, processing will resume on this stream.
+/// \param filename is the name of the script file
-/// \param filename is the name of the file containing the script
+/// \param newprompt is the command line prompt to associate with the file
-/// \param newprompt is the command line prompt
 void IfaceStatus::pushScript(const string &filename,const string &newprompt)
 
+{
+  ifstream *s = new ifstream(filename.c_str());
+  if (!*s)
+    throw IfaceParseError("Unable to open script file");
+}
+
+/// \brief Provide a new input stream to execute, with an associated command prompt
+///
+/// The new stream is added to a stack and becomes the primary source for parsing new commands.
+/// Once commands from the stream are exhausted, parsing will resume in the previous stream.
+/// \param iptr is the new input stream
+/// \param newprompt is the command line prompt to associate with the new stream
+void IfaceStatus::pushScript(istream *iptr,const string &newprompt)
+
 {
   promptstack.push_back(prompt);
   uint4 flags = 0;

Ghidra/Features/Decompiler/src/decompile/cpp/interface.hh

@@ -224,9 +224,10 @@ public:
   IfaceStatus(const string &prmpt,ostream &os,int4 mxhist=10);	///< Constructor
   virtual ~IfaceStatus(void);					///< Destructor
   void setErrorIsDone(bool val) { errorisdone = val; }	///< Set if processing should terminate on an error
-  virtual void pushScript(const string &filename,const string &newprompt);
+  void pushScript(const string &filename,const string &newprompt);
+  virtual void pushScript(istream *iptr,const string &newprompt);
   virtual void popScript(void);
-  void reset(void);	///< Pop any existing script streams and return to processing from the base stream
+  virtual void reset(void);	///< Pop any existing script streams and return to processing from the base stream
   int4 getNumInputStreamSize(void) const { return promptstack.size(); }	///< Get depth of script nesting
   void writePrompt(void) { *optr << prompt; }	///< Write the current command prompt to the current output stream
   void registerCom(IfaceCommand *fptr, const char *nm1,

Ghidra/Features/Decompiler/src/decompile/cpp/sleigh_arch.cc

@@ -16,8 +16,7 @@
 #include "sleigh_arch.hh"
 #include "inject_sleigh.hh"
 
-Sleigh *SleighArchitecture::last_sleigh = (Sleigh *)0;
+map<int4,Sleigh *> SleighArchitecture::translators;
-int4 SleighArchitecture::last_languageindex;
 vector<LanguageDescription> SleighArchitecture::description;
 
 FileManage SleighArchitecture::specpaths; // Global specfile manager
@@ -138,25 +137,23 @@ string SleighArchitecture::getDescription(void) const
 bool SleighArchitecture::isTranslateReused(void)
 
 {
-  if (last_sleigh == (Sleigh *)0) return false;
+  return (translators.find(languageindex) != translators.end());
-  if (last_languageindex == languageindex) return true;
-  delete last_sleigh;		// It doesn't match so free old Translate
-  last_sleigh = (Sleigh *)0;
-  return false;
 }
 
 Translate *SleighArchitecture::buildTranslator(DocumentStorage &store)
 
 {				// Build a sleigh translator
-  if (isTranslateReused()) {
+  map<int4,Sleigh *>::const_iterator iter;
-    last_sleigh->reset(loader,context);
+  Sleigh *sleigh;
-    return last_sleigh;
+  iter = translators.find(languageindex);
-  }
+  if (iter != translators.end()) {
-  else {
+    sleigh = (*iter).second;
-    last_sleigh = new Sleigh(loader,context);
+    sleigh->reset(loader,context);
-    last_languageindex = languageindex;
+    return sleigh;
-    return last_sleigh;
   }
+  sleigh = new Sleigh(loader,context);
+  translators[languageindex] = sleigh;
+  return sleigh;
 }
 
 PcodeInjectLibrary *SleighArchitecture::buildPcodeInjectLibrary(void)
@@ -463,9 +460,9 @@ void SleighArchitecture::scanForSleighDirectories(const string &rootpath)
 void SleighArchitecture::shutdown(void)
 
 {
-  if (last_sleigh != (Sleigh *)0) {
+  if (translators.empty()) return;	// Already cleared
-    delete last_sleigh;
+  for(map<int4,Sleigh *>::const_iterator iter=translators.begin();iter!=translators.end();++iter)
-    last_sleigh = (Sleigh *)0;
+    delete (*iter).second;
-  }
+  translators.clear();
   // description.clear();  // static vector is destroyed by the normal exit handler
 }

Ghidra/Features/Decompiler/src/decompile/cpp/sleigh_arch.hh

@@ -87,8 +87,7 @@ public:
 /// Generally a \e language \e id (i.e. x86:LE:64:default) is provided, then this
 /// object is able to automatically load in configuration and construct the Translate object.
 class SleighArchitecture : public Architecture {
-  static Sleigh *last_sleigh;				///< Last Translate object used by a SleighArchitecture
+  static map<int4,Sleigh *> translators;		///< Map from language index to instantiated translators
-  static int4 last_languageindex;			///< Index of the LanguageDescription associated with the last Translate object
   static vector<LanguageDescription> description;	///< List of languages we know about
   int4 languageindex;					///< Index (within LanguageDescription array) of the active language
   string filename;					///< Name of active load-image file

Ghidra/Features/Decompiler/src/decompile/cpp/test.cc

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 #include "test.hh"
-#include "testfunction.hh"
+#include "libdecomp.hh"
 
 vector<UnitTest *> UnitTest::tests;
 
@@ -44,6 +44,37 @@ void UnitTest::run(set<string> &testNames)
   std::cerr << passed << "/" << total << " tests passed." << std::endl;
 }
 
+/// Create list of the absolute path of all tests to be run
+/// \param dirname is a directory containing the XML test files
+/// \param testNames (if not empty) specifies particular tests to run
+/// \param testFiles will hold the resulting list of paths
+void gatherDataTests(const string &dirname,set<string> &testNames,vector<string> &testFiles)
+
+{
+  FileManage fileManage;
+
+  set<string> fullNames;
+  for(set<string>::iterator iter=testNames.begin();iter!=testNames.end();++iter) {
+    string val = dirname;
+    if (dirname.back() != '/')
+      val += '/';
+    val += *iter;
+    fullNames.insert(val);
+  }
+  fileManage.addDir2Path(dirname);
+  if (fullNames.empty()) {
+    fileManage.matchList(testFiles,".xml",true);	// Take all test files
+    return;
+  }
+  vector<string> allTestFiles;
+  fileManage.matchList(allTestFiles,".xml",true);
+  for(int4 i=0;i<allTestFiles.size();++i) {
+    if (fullNames.find(allTestFiles[i]) != fullNames.end()) {	// Take tests matching into list of basenames
+      testFiles.push_back(allTestFiles[i]);
+    }
+  }
+}
+
 int main(int argc, char **argv) {
   bool runUnitTests = true;
   bool runDataTests = true;
@@ -53,6 +84,7 @@ int main(int argc, char **argv) {
   set<string> unitTestNames;
   set<string> dataTestNames;
   string dirname("../datatests");
+  string sleighdirname("../../../../../../..");
   if (argc > 0) {
     string command(argv[0]);
     if (command == "-path") {
@@ -61,6 +93,22 @@ int main(int argc, char **argv) {
       argv += 2;
       argc -= 2;
     }
+    else if (command == "-sleighpath") {
+      sleighdirname = argv[1];
+      argv += 2;
+      argc -= 2;
+    }
+    else if (command == "-usesleighenv") {
+      const char *sleighhomepath = getenv("SLEIGHHOME");
+      if (sleighhomepath != (const char *)0) {
+        cout << "Using SLEIGHHOME=" << sleighhomepath << endl;
+        sleighdirname = sleighhomepath;
+      }
+      else
+        cout << "No SLEIGHHOME environment variable" << endl;
+      argv += 1;
+      argc -= 1;
+    }
   }
   if (argc > 0) {
     string command(argv[0]);
@@ -78,16 +126,13 @@ int main(int argc, char **argv) {
       cout << "USAGE: ghidra_test [-path <datatestdir>] [[unittests|datatests] [testname1 testname2 ...]]" << endl;
     }
   }
+  startDecompilerLibrary(sleighdirname.c_str());
   if (runUnitTests)
     UnitTest::run(unitTestNames);
   if (runDataTests) {
+    vector<string> testFiles;
+    gatherDataTests(dirname,dataTestNames,testFiles);
     cout << endl << endl;
-    const char *sleighhomepath = getenv("SLEIGHHOME");
+    FunctionTestCollection::runTestFiles(testFiles,cout);
-    if (sleighhomepath != (const char *)0)
-      cout << "Using SLEIGHHOME=" << sleighhomepath << endl;
-    else
-      cout << "No SLEIGHHOME environment variable" << endl;
-    startDecompilerLibrary(sleighhomepath);
-    FunctionTestCollection::runTestCollections(dirname,dataTestNames);
   }
 }

Ghidra/Features/Decompiler/src/decompile/cpp/testfunction.cc

@@ -13,8 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "testfunction.hh"
+#include "ifacedecomp.hh"
-#include "filemanage.hh"
 
 void FunctionTestProperty::startTest(void) const
 
@@ -57,8 +56,10 @@ void ConsoleCommands::readLine(string &line)
   pos += 1;
 }
 
-ConsoleCommands::ConsoleCommands(void) :
+/// \param s is the stream where command output is printed
-    IfaceStatus("> ", cout)
+/// \param comms is the list of commands to be issued
+ConsoleCommands::ConsoleCommands(ostream &s,vector<string> &comms) :
+    IfaceStatus("> ", s), commands(comms)
 {
   pos = 0;
   IfaceCapability::registerAllCommands(this);
@@ -67,14 +68,22 @@ ConsoleCommands::ConsoleCommands(void) :
 void ConsoleCommands::reset(void)
 
 {
-  commands.clear();
   pos = 0;
   inerror = false;
   done = false;
 }
 
+void FunctionTestCollection::clear(void)
+
+{
+  dcp->clearArchitecture();
+  commands.clear();
+  testList.clear();
+  console->reset();
+}
+
 /// \param el is the root \<script> tag
-void ConsoleCommands::restoreXml(const Element *el)
+void FunctionTestCollection::restoreXmlCommands(const Element *el)
 
 {
   const List &list(el->getChildren());
@@ -84,15 +93,6 @@ void ConsoleCommands::restoreXml(const Element *el)
     const Element *subel = *iter;
     commands.push_back(subel->getContent());
   }
-  pos = 0;
-}
-
-void FunctionTestCollection::clear(void)
-
-{
-  dcp->clearArchitecture();
-  testList.clear();
-  console.reset();
 }
 
 /// Instantiate an Architecture object
@@ -102,7 +102,7 @@ void FunctionTestCollection::buildProgram(DocumentStorage &docStorage)
   ArchitectureCapability *capa = ArchitectureCapability::getCapability("xml");
   if (capa == (ArchitectureCapability *)0)
     throw IfaceExecutionError("Missing XML architecture capability");
-  dcp->conf = capa->buildArchitecture("test", "", console.optr);
+  dcp->conf = capa->buildArchitecture("test", "", console->optr);
   string errmsg;
   bool iserror = false;
   try {
@@ -145,34 +145,53 @@ void FunctionTestCollection::passLineToTests(const string &line) const
 /// This is called after each test has been fed all lines of output.
 /// The result of each test is printed to the \e midStream, and then
 /// failures are written to the lateStream in order to see a summary.
-/// \param midStream is the stream write results to as the test is performed
 /// \param lateStream collects failures to display as a summary
-void FunctionTestCollection::evaluateTests(ostream &midStream,list<string> &lateStream) const
+void FunctionTestCollection::evaluateTests(list<string> &lateStream) const
 
 {
   list<FunctionTestProperty>::const_iterator iter;
   for(iter=testList.begin();iter!=testList.end();++iter) {
     numTestsApplied += 1;
     if ((*iter).endTest()) {
-      midStream << "Success -- " << (*iter).getName() << endl;
+      *console->optr << "Success -- " << (*iter).getName() << endl;
       numTestsSucceeded += 1;
     }
     else {
-      midStream << "FAIL -- " << (*iter).getName() << endl;
+      *console->optr << "FAIL -- " << (*iter).getName() << endl;
       lateStream.push_back((*iter).getName());
     }
   }
 }
 
-FunctionTestCollection::FunctionTestCollection(void)
+/// \param s is the stream where output is sent during tests
+FunctionTestCollection::FunctionTestCollection(ostream &s)
 
 {
-  dcp = (IfaceDecompData *)console.getData("decompile");
+  console = new ConsoleCommands(s,commands);
-  console.setErrorIsDone(true);
+  consoleOwner = true;
+  dcp = (IfaceDecompData *)console->getData("decompile");
+  console->setErrorIsDone(true);
   numTestsApplied = 0;
   numTestsSucceeded = 0;
 }
 
+FunctionTestCollection::FunctionTestCollection(IfaceStatus *con)
+
+{
+  console = con;
+  consoleOwner = false;
+  dcp = (IfaceDecompData *)console->getData("decompile");
+  numTestsApplied = 0;
+  numTestsSucceeded = 0;
+}
+
+FunctionTestCollection::~FunctionTestCollection(void)
+
+{
+  if (consoleOwner)
+    delete console;
+}
+
 /// Load the architecture based on the discovered \<binaryimage> tag.
 /// Collect the script commands and the specific tests.
 /// \param filename is the XML file holding the test data
@@ -194,7 +213,6 @@ void FunctionTestCollection::loadTest(const string &filename)
 void FunctionTestCollection::restoreXml(DocumentStorage &store,const Element *el)
 
 {
-  clear();
   const List &list(el->getChildren());
   List::const_iterator iter = list.begin();
   bool sawScript = false;
@@ -205,7 +223,7 @@ void FunctionTestCollection::restoreXml(DocumentStorage &store,const Element *el
     ++iter;
     if (subel->getName() == "script") {
       sawScript = true;
-      console.restoreXml(subel);
+      restoreXmlCommands(subel);
     }
     else if (subel->getName() == "stringmatch") {
       sawTests = true;
@@ -232,30 +250,29 @@ void FunctionTestCollection::restoreXml(DocumentStorage &store,const Element *el
 void FunctionTestCollection::restoreXmlOldForm(DocumentStorage &store,const Element *el)
 
 {
-  clear();
   throw IfaceParseError("Old format test not supported");
 }
 
 /// Run the script commands on the current program.
 /// Collect any bulk output, and run tests over the output.
 /// Report test failures back to the caller
-/// \param midStream is the output stream to write to during the test
 /// \param lateStream collects messages for a final summary
-void FunctionTestCollection::runTests(ostream &midStream,list<string> &lateStream)
+void FunctionTestCollection::runTests(list<string> &lateStream)
 
 {
+  ostream *origStream = console->optr;
   numTestsApplied = 0;
   numTestsSucceeded = 0;
   ostringstream midBuffer;		// Collect command console output
-  console.optr = &midBuffer;
+  console->optr = &midBuffer;
   ostringstream bulkout;
-  console.fileoptr = &bulkout;
+  console->fileoptr = &bulkout;
-  mainloop(&console);
+  mainloop(console);
-  console.optr = &midStream;
+  console->optr = origStream;
-  console.fileoptr = &midStream;
+  console->fileoptr = origStream;
-  if (console.isInError()) {
+  if (console->isInError()) {
-    midStream << "Error: Did not apply tests in " << fileName << endl;
+    *console->optr << "Error: Did not apply tests in " << fileName << endl;
-    midStream << midBuffer.str() << endl;
+    *console->optr << midBuffer.str() << endl;
     ostringstream fs;
     fs << "Execution failed for " << fileName;
     lateStream.push_back(fs.str());
@@ -281,63 +298,48 @@ void FunctionTestCollection::runTests(ostream &midStream,list<string> &lateStrea
     string line = result.substr(prevpos);	// Process final line without a newline char
     passLineToTests(line);
   }
-  evaluateTests(midStream, lateStream);
+  evaluateTests(lateStream);
 }
 
-/// Run through all XML files in the given directory, processing each in turn.
+/// Run through all XML files in the given list, processing each in turn.
-/// \param dirname is a directory containing the XML test files
+/// \param testFiles is the given list of test files
-/// \param testNames (if not empty) specifies particular tests to run
+/// \param s is the output stream to print results to
-void FunctionTestCollection::runTestCollections(const string &dirname,set<string> &testNames)
+void FunctionTestCollection::runTestFiles(const vector<string> &testFiles,ostream &s)
 
 {
-  FileManage fileManage;
-
-  set<string> fullNames;
-  for(set<string>::iterator iter=testNames.begin();iter!=testNames.end();++iter) {
-    string val = dirname;
-    if (dirname.back() != '/')
-      val += '/';
-    val += *iter;
-    fullNames.insert(val);
-  }
-  fileManage.addDir2Path(dirname);
-  vector<string> testFiles;
-  fileManage.matchList(testFiles,".xml",true);
-
   int4 totalTestsApplied = 0;
   int4 totalTestsSucceeded = 0;
   list<string> failures;
-  FunctionTestCollection testCollection;
+  FunctionTestCollection testCollection(s);
   for(int4 i=0;i<testFiles.size();++i) {
-    if (!fullNames.empty() && fullNames.find(testFiles[i]) == fullNames.end())
-      continue;
     try {
+      testCollection.clear();
       testCollection.loadTest(testFiles[i]);
-      testCollection.runTests(cout, failures);
+      testCollection.runTests(failures);
       totalTestsApplied += testCollection.getTestsApplied();
       totalTestsSucceeded += testCollection.getTestsSucceeded();
     } catch(IfaceParseError &err) {
       ostringstream fs;
       fs << "Error parsing " << testFiles[i] << ": " << err.explain;
-      cout << fs.str() << endl;
+      s << fs.str() << endl;
       failures.push_back(fs.str());
     } catch(IfaceExecutionError &err) {
       ostringstream fs;
       fs << "Error executing " << testFiles[i] << ": " << err.explain;
-      cout << fs.str() << endl;
+      s << fs.str() << endl;
       failures.push_back(fs.str());
     }
   }
 
-  cout << endl;
+  s << endl;
-  cout << "Total tests applied = " << totalTestsApplied << endl;
+  s << "Total tests applied = " << totalTestsApplied << endl;
-  cout << "Total passing tests = " << totalTestsSucceeded << endl;
+  s << "Total passing tests = " << totalTestsSucceeded << endl;
-  cout << endl;
+  s << endl;
   if (!failures.empty()) {
-    cout << "Failures: " << endl;
+    s << "Failures: " << endl;
     list<string>::const_iterator iter = failures.begin();
     for(int4 i=0;i<10;++i) {
-      cout << "  " << *iter << endl;
+      s << "  " << *iter << endl;
       ++iter;
       if (iter == failures.end()) break;
     }

Ghidra/Features/Decompiler/src/decompile/cpp/testfunction.hh

@@ -18,10 +18,13 @@
 #ifndef __TESTFUNCTION__
 #define __TESTFUNCTION__
 
-#include "libdecomp.hh"
+#include "ifaceterm.hh"
-#include <iostream>
+#include "error.hh"
+#include "xml.hh"
 #include <regex>
 
+class IfaceDecompData;
+
 /// \brief A single property to be searched for in the output of a function decompilation
 ///
 /// This is generally a regular expression run over the characters in the
@@ -43,14 +46,13 @@ public:
 
 /// \brief A console command run as part of a test sequence
 class ConsoleCommands : public IfaceStatus {
-  vector<string> commands;		///< Sequence of commands
+  vector<string> &commands;		///< Sequence of commands
   uint4 pos;				///< Position of next command to execute
   virtual void readLine(string &line);
 public:
-  ConsoleCommands(void);		///< Constructor
+  ConsoleCommands(ostream &s,vector<string> &comms);		///< Constructor
-  void reset(void);			///< Reset console for a new program
+  virtual void reset(void);		///< Reset console for a new program
   virtual bool isStreamFinished(void) const { return pos == commands.size(); }
-  void restoreXml(const Element *el);	///< Reconstruct the command from an XML tag
 };
 
 /// \brief A collection of tests around a single program/function
@@ -66,23 +68,30 @@ class FunctionTestCollection {
   IfaceDecompData *dcp;		///< Program data for the test collection
   string fileName;		///< Name of the file containing test data
   list<FunctionTestProperty> testList;	///< List of tests for this collection
-  ConsoleCommands console;	///< Decompiler console for executing scripts
+  vector<string> commands;	///< Sequence of commands for current test
+  IfaceStatus *console;		///< Decompiler console for executing scripts
+  bool consoleOwner;		///< Set to \b true if \b this object owns the console
   mutable int4 numTestsApplied;		///< Count of tests that were executed
   mutable int4 numTestsSucceeded;	///< Count of tests that passed
   void clear(void);		///< Clear any previous architecture and function
+  void restoreXmlCommands(const Element *el);	///< Reconstruct commands from an XML tag
   void buildProgram(DocumentStorage &store);	///< Build program (Architecture) from \<binaryimage> tag
   void startTests(void) const;	///< Initialize each FunctionTestProperty
   void passLineToTests(const string &line) const;	///< Let all tests analyze a line of the results
-  void evaluateTests(ostream &midStream,list<string> &lateStream) const;
+  void evaluateTests(list<string> &lateStream) const;
 public:
-  FunctionTestCollection(void);		///< Constructor
+  FunctionTestCollection(ostream &s);		///< Constructor
+  FunctionTestCollection(IfaceStatus *con);	///< Constructor with preexisting console
+  ~FunctionTestCollection(void);		///< Destructor
   int4 getTestsApplied(void) const { return numTestsApplied; }	///< Get the number of tests executed
   int4 getTestsSucceeded(void) const { return numTestsSucceeded; }	///< Get the number of tests that passed
+  int4 numCommands(void) const { return commands.size(); }	///< Get the number of commands in the current script
+  string getCommand(int4 i) const { return commands[i]; }	///< Get the i-th command
   void loadTest(const string &filename);	///< Load a test program, tests, and script
   void restoreXml(DocumentStorage &store,const Element *el);	///< Load tests from a \<decompilertest> tag.
   void restoreXmlOldForm(DocumentStorage &store,const Element *el);	///< Load tests from \<binaryimage> tag.
-  void runTests(ostream &midStream,list<string> &lateStream);	///< Run the script and perform the tests
+  void runTests(list<string> &lateStream);	///< Run the script and perform the tests
-  static void runTestCollections(const string &dirname,set<string> &testNames);	///< Run test files in a whole directory
+  static void runTestFiles(const vector<string> &testFiles,ostream &s);	///< Run tests for each listed file
 };
 
 #endif

Ghidra/Features/Decompiler/src/decompile/cpp/type.cc

@@ -130,9 +130,8 @@ Datatype *Datatype::nearestArrayedComponentBackward(uintb off,uintb *newoff,int4
   return (TypeArray *)0;
 }
 
-// Compare \b this with another data-type.
+/// Order \b this with another data-type, in a way suitable for the type propagation algorithm.
-/// 0 (equality) means the data-types are functionally equivalent (even if names differ)
+/// Bigger types come earlier. More specific types come earlier.
-/// Smaller types come earlier. More specific types come earlier.
 /// \param op is the data-type to compare with \b this
 /// \param level is maximum level to descend when recursively comparing
 /// \return negative, 0, positive depending on ordering of types
@@ -142,9 +141,11 @@ int4 Datatype::compare(const Datatype &op,int4 level) const
   return compareDependency(op);
 }
 
-/// Ordering of data-types for the main TypeFactory container.
+/// Sort data-types for the main TypeFactory container.  The sort needs to be based on
-/// Comparison only goes down one-level in the component structure,
+/// the data-type structure so that an example data-type, constructed outside the factory,
-/// before just comparing pointers.
+/// can be used to find the equivalent object inside the factory.  This means the
+/// comparison should not examine the data-type id. In practice, the comparison only needs
+/// to go down one level in the component structure before just comparing component pointers.
 /// \param op is the data-type to compare with \b this
 /// \return negative, 0, positive depending on ordering of types
 int4 Datatype::compareDependency(const Datatype &op) const
@@ -2101,6 +2102,22 @@ TypePointer *TypeFactory::getTypePointer(int4 s,Datatype *pt,uint4 ws)
   return (TypePointer *) findAdd(tmp);
 }
 
+/// The given name is attached, which distinguishes the returned data-type from
+/// other unnamed (or differently named) pointers that otherwise have the same attributes.
+/// \param s is the size of the pointer
+/// \param pt is the pointed-to data-type
+/// \param ws is the wordsize associated with the pointer
+/// \param n is the given name to attach to the pointer
+/// \return the TypePointer object
+TypePointer *TypeFactory::getTypePointer(int4 s,Datatype *pt,uint4 ws,const string &n)
+
+{
+  TypePointer tmp(s,pt,ws);
+  tmp.name = n;
+  tmp.id = Datatype::hashName(n);
+  return (TypePointer *) findAdd(tmp);
+}
+
 // Don't create more than a depth of 2, i.e. ptr->ptr->ptr->...
 /// \param s is the size of the pointer
 /// \param pt is the pointed-to data-type

Ghidra/Features/Decompiler/src/decompile/cpp/type.hh

@@ -118,7 +118,7 @@ public:
   virtual int4 numDepend(void) const { return 0; }	///< Return number of component sub-types
   virtual Datatype *getDepend(int4 index) const { return (Datatype *)0; }	///< Return the i-th component sub-type
   virtual void printNameBase(ostream &s) const { if (!name.empty()) s<<name[0]; } ///< Print name as short prefix
-  virtual int4 compare(const Datatype &op,int4 level) const; ///< Compare for functional equivalence
+  virtual int4 compare(const Datatype &op,int4 level) const; ///< Order types for propagation
   virtual int4 compareDependency(const Datatype &op) const; ///< Compare for storage in tree structure
   virtual Datatype *clone(void) const=0;	///< Clone the data-type
   virtual void saveXml(ostream &s) const;	///< Serialize the data-type to XML
@@ -443,6 +443,7 @@ public:
   TypeCode *getTypeCode(void);					///< Get an "anonymous" function data-type
   TypePointer *getTypePointerStripArray(int4 s,Datatype *pt,uint4 ws);	///< Construct a pointer data-type, stripping an ARRAY level
   TypePointer *getTypePointer(int4 s,Datatype *pt,uint4 ws);	///< Construct an absolute pointer data-type
+  TypePointer *getTypePointer(int4 s,Datatype *pt,uint4 ws,const string &n);	///< Construct a named pointer data-type
   TypePointer *getTypePointerNoDepth(int4 s,Datatype *pt,uint4 ws);	///< Construct a depth limited pointer data-type
   TypeArray *getTypeArray(int4 as,Datatype *ao);		///< Construct an array data-type
   TypeStruct *getTypeStruct(const string &n);			///< Create an (empty) structure

Ghidra/Features/Decompiler/src/decompile/cpp/xml_arch.cc

@@ -24,6 +24,12 @@ XmlArchitectureCapability::XmlArchitectureCapability(void)
   name = "xml";
 }
 
+XmlArchitectureCapability::~XmlArchitectureCapability(void)
+
+{
+  SleighArchitecture::shutdown();
+}
+
 Architecture *XmlArchitectureCapability::buildArchitecture(const string &filename,const string &target,ostream *estream)
 
 {

Ghidra/Features/Decompiler/src/decompile/cpp/xml_arch.hh

@@ -25,6 +25,7 @@ class XmlArchitectureCapability : public ArchitectureCapability {
   XmlArchitectureCapability(const XmlArchitectureCapability &op2);	///< Not implemented
   XmlArchitectureCapability &operator=(const XmlArchitectureCapability &op2);	///< Not implemented
 public:
+  virtual ~XmlArchitectureCapability(void);
   virtual Architecture *buildArchitecture(const string &filename,const string &target,ostream *estream);
   virtual bool isFileMatch(const string &filename) const;
   virtual bool isXmlMatch(Document *doc) const;

Ghidra/Features/Decompiler/src/decompile/unittests/testtypes.cc

@@ -0,0 +1,183 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "architecture.hh"
+#include "grammar.hh"
+#include "test.hh"
+#include <iostream>
+
+Architecture *glb;
+TypeFactory *types;
+CastStrategy *strategy;
+Funcdata *dummyFunc;
+
+class TypeTestEnvironment {
+  Architecture *g;
+public:
+  TypeTestEnvironment(void);
+  ~TypeTestEnvironment(void);
+  static void build(void);
+};
+
+TypeTestEnvironment theEnviron;
+
+TypeTestEnvironment::TypeTestEnvironment(void)
+
+{
+  g = (Architecture *)0;
+}
+
+void TypeTestEnvironment::build(void)
+
+{
+  if (theEnviron.g != (Architecture *)0) return;
+  ArchitectureCapability *xmlCapability = ArchitectureCapability::getCapability("xml");
+  istringstream s(
+      "<binaryimage arch=\"x86:LE:64:default:gcc\"></binaryimage>"
+  );
+  DocumentStorage store;
+  Document *doc = store.parseDocument(s);
+  store.registerTag(doc->getRoot());
+
+  theEnviron.g = xmlCapability->buildArchitecture("", "", &cout);
+  theEnviron.g->init(store);
+
+  glb = theEnviron.g;
+  types = glb->types;
+  strategy = glb->print->getCastStrategy();
+  Address addr(glb->getDefaultCodeSpace(),0x1000);
+  dummyFunc = glb->symboltab->getGlobalScope()->addFunction(addr, "dummy")->getFunction();
+  dummyFunc->setHighLevel();
+}
+
+TypeTestEnvironment::~TypeTestEnvironment(void)
+
+{
+  if (g != (Architecture *)0)
+    delete g;
+}
+
+Datatype *parse(const string &text) {
+  istringstream s(text);
+  string unused;
+  return parse_type(s,unused,glb);
+}
+
+bool castPrinted(OpCode opc,Datatype *t1,Datatype *t2) {
+  TypeOp *inst = glb->inst[opc];
+  PcodeOp *op;
+  Address addr(glb->getDefaultCodeSpace(),0x1000);
+  if ((inst->getFlags() & PcodeOp::unary)!=0) {
+    op = dummyFunc->newOp(1, addr);
+    Varnode *vn1 = dummyFunc->newUnique(t2->getSize(), t2);
+    Varnode *outvn = dummyFunc->newUniqueOut(t1->getSize(), op);
+    outvn->updateType(t1, true, true);
+    dummyFunc->opSetOpcode(op, opc);
+    dummyFunc->opSetInput(op, vn1, 0);
+  }
+  else {
+    op = dummyFunc->newOp(2, addr);
+    Varnode *vn1 = dummyFunc->newUnique(t1->getSize(), t1);
+    Varnode *vn2 = dummyFunc->newUnique(t2->getSize(), t2);
+    dummyFunc->opSetOpcode(op, opc);
+    dummyFunc->opSetInput(op, vn1, 0);
+    dummyFunc->opSetInput(op, vn2, 1);
+    dummyFunc->newUniqueOut(1, op);
+  }
+  return (inst->getInputCast(op, 0, strategy) != (Datatype *)0);
+}
+
+TEST(cast_basic) {
+  TypeTestEnvironment::build();
+  ASSERT(castPrinted(CPUI_COPY,parse("int4"),parse("int2")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("int4"),parse("uint4")));
+  ASSERT(castPrinted(CPUI_COPY,parse("int4 *"),parse("uint8")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("int1"),parse("bool")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("xunknown4"),parse("uint4")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("int4"),parse("xunknown4")));
+  ASSERT(castPrinted(CPUI_COPY,parse("int4"),parse("float4")));
+  ASSERT(castPrinted(CPUI_COPY,parse("int1 var[4]"),parse("uint4")));
+  Datatype *typedefInt = types->getBase(4,TYPE_INT,"myint4");
+  ASSERT(!castPrinted(CPUI_COPY,typedefInt,parse("int4")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("char"),parse("int1")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("uint1"),parse("char")));
+}
+
+TEST(cast_pointer) {
+  TypeTestEnvironment::build();
+  ASSERT(castPrinted(CPUI_COPY,parse("uint4 *"),parse("int4 *")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("void *"),parse("float4 *")));
+  ASSERT(castPrinted(CPUI_COPY,parse("int2 *"),parse("void *")));
+  Datatype *typedefInt = types->getBase(4,TYPE_INT,"myint4");
+  Datatype *typedefPtr = types->getTypePointer(8,typedefInt,1);
+  ASSERT(!castPrinted(CPUI_COPY,typedefPtr,parse("int4 *")));
+  ASSERT(castPrinted(CPUI_COPY,parse("bool **"),parse("int1 **")));
+  parse("struct structone { int4 a; int4 b; }");
+  parse("struct structtwo { int4 a; int4 b; }");
+  ASSERT(castPrinted(CPUI_COPY,parse("structone *"),parse("structtwo *")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("xunknown4 *"),parse("int4 *")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("uint4 *"),parse("xunknown4 *")));
+  ASSERT(!castPrinted(CPUI_COPY,parse("char *"),parse("int1 *")));
+  ASSERT(castPrinted(CPUI_COPY,parse("uint1 *"),parse("char *")));
+  Datatype *ptrNamed = types->getTypePointer(8,parse("int4"),1,"myptrint4");
+  ASSERT(!castPrinted(CPUI_COPY,parse("int4 *"),ptrNamed));
+}
+
+TEST(cast_enum) {
+  TypeTestEnvironment::build();
+  Datatype *enum1 = parse("enum enumone { ONE=1, TWO=2 }");
+  ASSERT(!castPrinted(CPUI_COPY,parse("int8"),enum1));
+  ASSERT(!castPrinted(CPUI_COPY,parse("uint8 *"),parse("enumone *")));
+  ASSERT(!castPrinted(CPUI_COPY,enum1,parse("uint8")));
+}
+
+TEST(cast_compare) {
+  TypeTestEnvironment::build();
+  ASSERT(castPrinted(CPUI_INT_LESS,parse("int4"),parse("int4")));
+  ASSERT(!castPrinted(CPUI_INT_LESS,parse("uint4"),parse("uint4")));
+  ASSERT(!castPrinted(CPUI_INT_LESS,parse("int4 *"),parse("int4 *")));
+  ASSERT(castPrinted(CPUI_INT_SLESS,parse("uint4"),parse("uint4")));
+  ASSERT(!castPrinted(CPUI_INT_SLESS,parse("int4"),parse("int4")));
+  ASSERT(castPrinted(CPUI_INT_EQUAL,parse("uint8"),parse("int4 *")));
+  ASSERT(!castPrinted(CPUI_INT_EQUAL,parse("int4 *"),parse("uint8")));
+  ASSERT(!castPrinted(CPUI_INT_NOTEQUAL,parse("int4"),parse("uint4")));
+  ASSERT(!castPrinted(CPUI_INT_NOTEQUAL,parse("uint4"),parse("int4")));
+  ASSERT(castPrinted(CPUI_INT_EQUAL,parse("int4"),parse("float4")));
+}
+
+TEST(type_ordering) {
+  TypeTestEnvironment::build();
+  ASSERT(parse("uint4")->compare(*parse("int4"),10) < 0);
+  Datatype *intTypeDef = types->getBase(4,TYPE_INT,"myint4");
+  ASSERT_NOT_EQUALS(parse("int4"),intTypeDef);
+  ASSERT(parse("int4")->compareDependency(*intTypeDef) == 0);
+  ASSERT(parse("int1")->compare(*parse("char"),10) < 0);
+  ASSERT(parse("wchar2")->compare(*parse("int2"),10) < 0);
+  ASSERT(parse("wchar4")->compare(*parse("int4"),10) < 0);
+  ASSERT(parse("uint1")->compare(*parse("char"),10) < 0);
+  Datatype *enum1 = parse("enum enum2 { ONE=1, TWO=2 }");
+  ASSERT(enum1->compare(*parse("int8"),10) < 0);
+  Datatype *struct1 = parse("struct struct1 { int4 a; int4 b; }");
+  Datatype *struct2 = parse("struct struct2 { int4 a; int4 b; }");
+  ASSERT_NOT_EQUALS(struct1,struct2);
+  ASSERT(struct1->compareDependency(*struct2) == 0);
+  ASSERT(parse("uint4")->compare(*parse("uint2"),10) < 0);
+  ASSERT(parse("float8")->compare(*parse("float4"),10) < 0);
+  ASSERT(parse("bool")->compare(*parse("uint1"),10) < 0);
+  ASSERT(parse("uint4 *")->compare(*parse("int4 *"),10) < 0);
+  ASSERT(parse("enum2 *")->compare(*parse("int8 *"),10) < 0);
+  ASSERT(parse("int4 *")->compare(*parse("void *"),10) < 0);
+  ASSERT(parse("int2 *")->compare(*parse("xunknown2 *"),10) < 0);
+}