Refactor namespaces to use ids

Ghidra/Features/Decompiler/src/decompile/cpp/database_ghidra.cc

@@ -35,46 +35,66 @@ ScopeGhidra::~ScopeGhidra(void)
 /// creates a dedicated Scope object to hold such a Symbol. The immediate
 /// parent for the new Scope must already exist.
 /// \param nm is the name of the new \e namespace
+/// \param id is the Ghidra specific id associated with the namespace
 /// \param par is the parent Scope
 /// \return the new \e namespace Scope
-Scope *ScopeGhidra::createNewScope(const string &nm,Scope *par) const
+Scope *ScopeGhidra::createNewScope(const string &nm,uint8 id,Scope *par) const
 
 {
-  Scope *newscope = new ScopeGhidraNamespace(nm,ghidra);
+  Scope *newscope = par->resolveScope(nm);
+  if (newscope != (Scope *)0) {
+    // Its possible because of the way that we merge root namespaces that
+    // Ghidra has distinct namespace ids for namespaces that we want merged
+    // So we return any existing namespace even though the id is different.
+    return newscope;
+  }
+  newscope = new ScopeGhidraNamespace(nm,id,ghidra);
   ghidra->symboltab->attachScope(newscope,par);
-//   Document *doc = ghidra->getScopeProperties(newscope);
-//   if (doc == (Document *)0)
-//     throw LowlevelError("Bad getScopeProperties response");
-//   const Element *root = doc->getRoot();
-//   if (root->getName() == "rangelist") {
-//     RangeList newrangetree;
-//     newrangetree.restoreXml(root,ghidra);
-//     ghidra->symboltab->set_range(newscope,newrangetree);
-//   }
-//   delete doc;
+  if (id != 0)
+    namespaceMap[id] = newscope;
   return newscope;
 }
 
-/// The Ghidra client reports a \e namespace path associated with
-/// Symbol. Determine if this Scope already exists in the cache and built
+/// The Ghidra client reports a \e namespace id associated with
+/// Symbol. Determine if a matching \e namespac Scope already exists in the cache and build
 /// it if it isn't. This may mean creating a new \e namespace Scope.
-/// \param path is absolute path to the desired Scope
-/// \return the Scope matching the path.
-Scope *ScopeGhidra::reresolveScope(const vector<string> &path) const
+/// \param id is the ID associated with the Ghidra namespace
+/// \return the Scope matching the id.
+Scope *ScopeGhidra::reresolveScope(uint8 id) const
 
 {
-  if (path.size()==1) return cache;
-  // Get pointer to ourselves (which is not const)
-  Scope *curscope = glb->symboltab->getGlobalScope();
-  int4 i;
-  for(i=1;i<path.size();++i) {
-    Scope *nextscope = curscope->resolveScope(path[i]);
-    if (nextscope == (Scope *)0) break;
-    curscope = nextscope;
+  if (id == 0) return cache;
+  map<uint8,Scope *>::const_iterator miter = namespaceMap.find(id);
+  if (miter != namespaceMap.end())
+    return (*miter).second;		// Scope was previously cached
+
+  Document *doc = ghidra->getNamespacePath(id);
+  if (doc == (Document *)0)
+    throw LowlevelError("Could not get namespace info");
+
+  Scope *curscope = glb->symboltab->getGlobalScope();	// Get pointer to ourselves (which is not const)
+  try {
+    const List &list(doc->getRoot()->getChildren());
+    List::const_iterator iter = list.begin();
+    ++iter;		// Skip element describing the root scope
+    while(iter != list.end()) {
+      const Element *el = *iter;
+      ++iter;
+      uint8 scopeId;
+      istringstream s(el->getAttributeValue("id"));
+      s.unsetf(ios::dec | ios::hex | ios::oct);
+      s >> scopeId;
+      miter = namespaceMap.find(scopeId);
+      if (miter == namespaceMap.end())
+	curscope = createNewScope(el->getContent(), scopeId, curscope);
+      else
+	curscope = (*miter).second;
     }
-  while(i != path.size()) {
-    curscope = createNewScope(path[i],curscope);
-    i += 1;
+    delete doc;
+  }
+  catch(LowlevelError &err) {
+    delete doc;
+    throw err;
   }
   return curscope;
 }
@@ -126,16 +146,14 @@ Symbol *ScopeGhidra::dump2Cache(Document *doc) const
   }
 
   List::const_iterator iter = el->getChildren().begin();
-  // The first subnode must be scope information
-  el = *iter;
-  vector<string> path;
-  const List &list2(el->getChildren());
-  List::const_iterator iter2;
-  for(iter2=list2.begin();iter2!=list2.end();++iter2)
-    path.push_back( (*iter2)->getContent() );
+  uint8 scopeId;
+  {
+    istringstream s(el->getAttributeValue("id"));
+    s.unsetf(ios::dec | ios::hex | ios::oct);
+    s >> scopeId;
+  }
 
-  Scope *scope = reresolveScope(path);
-  ++iter;			// The second part is a <mapsym>
+  Scope *scope = reresolveScope(scopeId);
   el = *iter;
   try {
     sym = scope->addMapSym(el);
@@ -255,6 +273,7 @@ void ScopeGhidra::clear(void)
 {
   cache->clear();
   holes.clear();
+  namespaceMap.clear();
   if (cacheDirty) {
     ghidra->symboltab->setProperties(flagbaseDefault); // Restore database properties to defaults
     cacheDirty = false;

Ghidra/Features/Decompiler/src/decompile/cpp/database_ghidra.hh

@@ -36,14 +36,15 @@ class ScopeGhidra : public Scope {
   ArchitectureGhidra *ghidra;		///< Architecture and connection to the Ghidra client
   mutable ScopeInternal *cache;		///< An internal cache of previously fetched Symbol objects
   mutable RangeList holes;		///< List of (queried) memory ranges with no Symbol in them
+  mutable map<uint8,Scope *> namespaceMap;	///< Map from id to formal global namespace objects
   vector<int4> spacerange;		///< List of address spaces that are in the global range
   partmap<Address,uint4> flagbaseDefault;	///< Default boolean properties on memory
   mutable bool cacheDirty;		///< Is flagbaseDefault different from cache
   Symbol *dump2Cache(Document *doc) const;			///< Parse a response into the cache
   Symbol *removeQuery(const Address &addr) const;		///< Process a query that missed the cache
   void processHole(const Element *el) const;			///< Process a response describing a hole
-  Scope *createNewScope(const string &nm,Scope *par) const;	///< Create a global \e namespace Scope
-  Scope *reresolveScope(const vector<string> &path) const;	///< Find the Scope that will contain a result Symbol
+  Scope *createNewScope(const string &nm,uint8 id,Scope *par) const;	///< Create a global \e namespace Scope
+  Scope *reresolveScope(uint8 id) const;	///< Find the Scope that will contain a result Symbol
   virtual void addRange(AddrSpace *spc,uintb first,uintb last);
   virtual void removeRange(AddrSpace *spc,uintb first,uintb last) {
     throw LowlevelError("remove_range should not be performed on ghidra scope");
@@ -124,11 +125,14 @@ public:
 /// be a ScopeGhidra.  This will query the Ghidra client on behalf of the \e namespace and
 /// register any new symbols with \b this Scope.
 class ScopeGhidraNamespace : public ScopeInternal {
+  uint8 scopeId;			///< Internal id allowing Ghidra client to reference formal namespaces
   virtual SymbolEntry *addMapInternal(Symbol *sym,uint4 exfl,const Address &addr,int4 off,int4 sz,
 				      const RangeList &uselim);
 public:
-  ScopeGhidraNamespace(const string &nm,Architecture *g)
-    : ScopeInternal(nm,g) {}		///< Constructor
+  ScopeGhidraNamespace(const string &nm,uint8 id,Architecture *g)
+    : ScopeInternal(nm,g) { scopeId = id; }		///< Constructor
+
+  uint8 getId(void) const { return scopeId; }		///< Get the Ghidra specific id
 };
 
 #endif

Ghidra/Features/Decompiler/src/decompile/cpp/ghidra_arch.cc

@@ -515,6 +515,25 @@ Document *ArchitectureGhidra::getExternalRefXML(const Address &addr)
   return readXMLAll(sin);
 }
 
+/// Ask the Ghidra client to list all namespace elements between the global root
+/// and the namespace of the given id. The client should return a \<parent> tag with
+/// a \<val> child for each namespace in the path.
+/// \param id is the given id of the namespace to resolve
+/// \return the XML document
+Document *ArchitectureGhidra::getNamespacePath(uint8 id)
+
+{
+  sout.write("\000\000\001\004",4);
+  writeStringStream(sout,"getNamespacePath");
+  sout.write("\000\000\001\016",4); // Beginning of string header
+  sout << hex << id;
+  sout.write("\000\000\001\017",4);
+  sout.write("\000\000\001\005",4);
+  sout.flush();
+
+  return readXMLAll(sin);
+}
+
 /// Get the name of the primary symbol at the given address.
 /// This is used to fetch within function \e labels. Only a name is returned.
 /// \param addr is the given address

Ghidra/Features/Decompiler/src/decompile/cpp/ghidra_arch.hh

@@ -92,6 +92,7 @@ public:
   uint1 *getPcodePacked(const Address &addr);			///< Get p-code for a single instruction
   Document *getMappedSymbolsXML(const Address &addr);		///< Get symbols associated with the given address
   Document *getExternalRefXML(const Address &addr);		///< Retrieve a description of an external function
+  Document *getNamespacePath(uint8 id);				///< Get a description of a namespace path
   string getCodeLabel(const Address &addr);			///< Retrieve a label at the given address
   Document *getType(const string &name,uint8 id);		///< Retrieve a data-type description for the given name and id
   Document *getComments(const Address &fad,uint4 flags);	///< Retrieve comments for a particular function

Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileCallback.java

@@ -528,6 +528,15 @@ public class DecompileCallback {
 		return sym.getName();
 	}
 
+	private Namespace getNameSpaceByID(long id) {
+		Symbol namespaceSym = program.getSymbolTable().getSymbol(id);
+		Object namespace = namespaceSym.getObject();
+		if (namespace instanceof Namespace) {
+			return (Namespace) namespace;
+		}
+		return null;
+	}
+
 	private String getNamespacePrefix(Namespace ns) {
 		if (ns.getID() == Namespace.GLOBAL_NAMESPACE_ID) {
 			return null;
@@ -543,6 +552,18 @@ public class DecompileCallback {
 		return name;
 	}
 
+	/**
+	 * Return an XML description of the formal namespace path to the given namespace
+	 * @param id is the ID of the given namespace
+	 * @return a parent XML tag
+	 */
+	public String getNamespacePath(long id) {
+		Namespace namespace = getNameSpaceByID(id);
+		StringBuilder buf = new StringBuilder();
+		HighFunction.createNamespaceTag(buf, namespace, true);
+		return buf.toString();
+	}
+
 	private void generateHeaderCommentXML(Function func, StringBuilder buf) {
 		Address addr = func.getEntryPoint();
 		String text = listing.getComment(CodeUnit.PLATE_COMMENT, addr);
@@ -802,16 +823,17 @@ public class DecompileCallback {
 	}
 
 	private String buildResult(HighSymbol highSymbol, Namespace namespc) {
-		StringBuilder res = new StringBuilder();
-		res.append("<result>\n");
-		res.append("<parent>\n");
-		if (namespc == null) {
-			res.append("<val/>"); // Assume global scope
+		long namespaceId;
+		if (namespc == null || namespc instanceof Library) {
+			namespaceId = Namespace.GLOBAL_NAMESPACE_ID;
 		}
 		else {
-			HighFunction.createNamespaceTag(res, namespc);
+			namespaceId = namespc.getID();
 		}
-		res.append("</parent>\n");
+		StringBuilder res = new StringBuilder();
+		res.append("<result");
+		SpecXmlUtils.encodeUnsignedIntegerAttribute(res, "id", namespaceId);
+		res.append(">\n");
 		if (debug != null) {
 			StringBuilder res2 = new StringBuilder();
 			HighSymbol.buildMapSymXML(res2, highSymbol);

Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileDebug.java

@@ -522,18 +522,19 @@ public class DecompileDebug {
 		}
 		while (scopename != null) {
 			StringBuilder datahead = new StringBuilder();
+			Namespace parentNamespace;
 			datahead.append("<scope");
 			// Force globalnamespace to have blank name
-			if (scopename != globalnamespace) {
+			if (scopename != globalnamespace && !(scopename instanceof Library)) {
 				SpecXmlUtils.xmlEscapeAttribute(datahead, "name", scopename.getName());
+				parentNamespace = scopename.getParentNamespace();
 			}
 			else {
 				SpecXmlUtils.encodeStringAttribute(datahead, "name", "");
+				parentNamespace = null;
 			}
 			datahead.append(">\n");
-			datahead.append("<parent>\n");
-			HighFunction.createNamespaceTag(datahead, scopename.getParentNamespace());
-			datahead.append("</parent>\n");
+			HighFunction.createNamespaceTag(datahead, parentNamespace, false);
 			if (scopename != globalnamespace) {
 				datahead.append("<rangeequalssymbols/>\n");
 			}

Ghidra/Features/Decompiler/src/main/java/ghidra/app/decompiler/DecompileProcess.java

@@ -314,6 +314,9 @@ public class DecompileProcess {
 							case 'M':
 								getMappedSymbolsXML();			// getMappedSymbolsXML
 								break;
+							case 'N':
+								getNamespacePath();
+								break;
 							case 'P':
 								getPcodePacked();				// getPacked
 								break;
@@ -711,6 +714,17 @@ public class DecompileProcess {
 		write(query_response_end);
 	}
 
+	private void getNamespacePath() throws IOException {
+		String idString = readQueryString();
+		long id = Long.parseLong(idString, 16);
+		String res = callback.getNamespacePath(id);
+		write(query_response_start);
+		if ((res != null) && (res.length() != 0)) {
+			writeString(res);
+		}
+		write(query_response_end);
+	}
+
 	private void getExternalRefXML() throws IOException {
 		String refaddr = readQueryString();
 		String res = callback.getExternalRefXML(refaddr);

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/HighFunction.java

@@ -618,14 +618,21 @@ public class HighFunction extends PcodeSyntaxTree {
 		}
 	}
 
-	static public void createNamespaceTag(StringBuilder buf, Namespace namespc) {
-		if (namespc == null) {
-			return;
-		}
-		ArrayList<String> arr = new ArrayList<String>();
-		Namespace curspc = namespc;
+	/**
+	 * Append an XML &lt;parent&gt; tag to the buffer describing the formal path elements
+	 * from the root (global) namespace up to the given namespace
+	 * @param buf is the buffer to write to
+	 * @param namespace is the namespace being described
+	 * @param includeId is true if the XML tag should include namespace ids
+	 */
+	static public void createNamespaceTag(StringBuilder buf, Namespace namespace,
+			boolean includeId) {
+		buf.append("<parent>\n");
+		if (namespace != null) {
+			ArrayList<Namespace> arr = new ArrayList<Namespace>();
+			Namespace curspc = namespace;
 			while (curspc != null) {
-			arr.add(0, curspc.getName());
+				arr.add(0, curspc);
 				if (curspc instanceof Library) {
 					break;		// Treat library namespace as root
 				}
@@ -633,11 +640,18 @@ public class HighFunction extends PcodeSyntaxTree {
 			}
 			buf.append("<val/>\n"); // Force global scope to have empty name
 			for (int i = 1; i < arr.size(); ++i) {
-			buf.append("<val>");
-			SpecXmlUtils.xmlEscape(buf, arr.get(i));
+				Namespace curScope = arr.get(i);
+				buf.append("<val");
+				if (includeId) {
+					SpecXmlUtils.encodeUnsignedIntegerAttribute(buf, "id", curScope.getID());
+				}
+				buf.append('>');
+				SpecXmlUtils.xmlEscape(buf, curScope.getName());
 				buf.append("</val>\n");
 			}
 		}
+		buf.append("</parent>\n");
+	}
 
 	/**
 	 * @param tagname -- Name of tag to search for

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/LocalSymbolMap.java

@@ -338,9 +338,7 @@ public class LocalSymbolMap {
 		resBuf.append("<scope");
 		SpecXmlUtils.xmlEscapeAttribute(resBuf, "name", func.getFunction().getName());
 		resBuf.append(">\n");
-		resBuf.append("<parent>\n");
-		HighFunction.createNamespaceTag(resBuf, namespace);
-		resBuf.append("</parent>\n");
+		HighFunction.createNamespaceTag(resBuf, namespace, false);
 		resBuf.append("<rangelist/>\n");	// Empty address range
 		resBuf.append("<symbollist>\n");
 		Iterator<HighSymbol> iter = symbolMap.values().iterator();