name to address space map

shortcut to address space map
more adjustments to shortcuts
allow null AddrSpace pointer in raw baselist
holes in the space indices
almost working
GT-2873 decompiler, other, and overlays
GT-2873 added OTHER space to java sleigh compiler, fixed decompiler
exception
isOtherSpace method
isOtherSpace java, addressing code review comments
GT-2873 added null check in decompiler reset
GT-2873 code review changes
Read and write space_other tag in SLA files
Version number for .sla file
GT-2873 fixups after merge
GT-2873 renamed Sparc registers: OTHER->OTHERWIN, WINWSTATE->WSTATE
GT-2873 added option in AddressInput to control OTHER space visibility
GT-2873 OTHER space now global
GT-2873 fixing comments refering to decompiler code in BasicCompilerSpec

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/SleighLanguage.java

@@ -52,6 +52,8 @@ import utilities.util.FileUtilities;
 
 public class SleighLanguage implements Language {
 
+	public static final int SLA_FORMAT_VERSION = 2;	// What format of the .sla file this expects
+													// This value should always match SleighBase.SLA_FORMAT_VERSION
 	private Map<CompilerSpecID, SleighCompilerSpecDescription> compilerSpecDescriptions;
 	private HashMap<CompilerSpecID, BasicCompilerSpec> compilerSpecs;
 	private List<InjectPayloadSleigh> additionalInject = null;
@@ -805,6 +807,10 @@ public class SleighLanguage implements Language {
 
 	private void restoreXml(XmlPullParser parser) throws UnknownInstructionException {
 		XmlElement el = parser.start("sleigh");
+		int version = SpecXmlUtils.decodeInt(el.getAttribute("version"));
+		if (version != SLA_FORMAT_VERSION) {
+			throw new SleighException(".sla file for " + getLanguageID() + " has the wrong format");
+		}
 		boolean isBigEndian = SpecXmlUtils.decodeBoolean(el.getAttribute("bigendian"));
 		// check the instruction endianess, not the program data endianess
 		if (isBigEndian ^ description.getInstructionEndian().isBigEndian()) {
@@ -840,8 +846,16 @@ public class SleighLanguage implements Language {
 		// Slot zero is always the constant space
 		AddressSpace constspc = new GenericAddressSpace("const", 64, AddressSpace.TYPE_CONSTANT, 0);
 		spacetable.put("const", constspc);
+		//spacetable.put("OTHER", AddressSpace.OTHER_SPACE);
 		default_space = null;
-		XmlElement subel;
+		XmlElement subel = parser.peek();
+		if (subel.getName().equals("space_other")) {	// tag must be present
+			parser.discardSubTree();	// We don't process it
+			// Instead the ProgramAddressFactory maps in the static OTHER_SPACE automatically 
+		}
+		else {
+			throw new SleighException(".sla file missing required OTHER space tag");
+		}
 		while ((subel = parser.softStart("space", "space_unique")) != null) {
 			String name = subel.getAttribute("name");
 			int index = SpecXmlUtils.decodeInt(subel.getAttribute("index"));
@@ -1319,14 +1333,13 @@ public class SleighLanguage implements Language {
 		int delay;
 		boolean physical;
 		boolean global;
-		int index = 1;
+
 		for (AddressSpace element : spclist) {
-			if ((element instanceof OverlayAddressSpace) &&
-				(element.getType() == AddressSpace.TYPE_RAM)) {
+			if ((element instanceof OverlayAddressSpace)) {
 				OverlayAddressSpace ospace = (OverlayAddressSpace) element;
 				resBuf.append("<space_overlay");
 				SpecXmlUtils.encodeStringAttribute(resBuf, "name", ospace.getName());
-				SpecXmlUtils.encodeSignedIntegerAttribute(resBuf, "index", index++);
+				SpecXmlUtils.encodeSignedIntegerAttribute(resBuf, "index", ospace.getUnique());
 				SpecXmlUtils.encodeStringAttribute(resBuf, "base",
 					ospace.getOverlayedSpace().getName());
 				resBuf.append("/>\n");
@@ -1351,12 +1364,18 @@ public class SleighLanguage implements Language {
 					physical = true;
 					global = false;
 					break;
+				case AddressSpace.TYPE_OTHER:
+					tag = "space_other";
+					delay = 0;
+					physical = true;
+					global = true;
+					break;
 				default:
 					continue;
 			}
 			resBuf.append("<").append(tag);
 			SpecXmlUtils.encodeStringAttribute(resBuf, "name", element.getName());
-			SpecXmlUtils.encodeSignedIntegerAttribute(resBuf, "index", index++);
+			SpecXmlUtils.encodeSignedIntegerAttribute(resBuf, "index", element.getUnique());
 
 			int size = element.getSize(); // Size in bits
 			if (size == 20) {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/sleighbase/SleighBase.java

@@ -31,6 +31,8 @@ public abstract class SleighBase extends Translate implements NamedSymbolProvide
 
 	// NOTE: restoreXml method removed as it is only used by the decompiler's implementation
 
+	public static final int SLA_FORMAT_VERSION = 2;	// What format of the .sla file this produces
+													// This value should always match SleighLanguage.SLA_FORMAT_VERSION
 	private VectorSTL<String> userop = new VectorSTL<>();
 	private address_set varnode_xref = new address_set(); // Cross-reference registers by address
 	protected SubtableSymbol root;
@@ -187,6 +189,7 @@ public abstract class SleighBase extends Translate implements NamedSymbolProvide
 
 	public void saveXml(PrintStream s) {
 		s.append("<sleigh");
+		XmlUtils.a_v_i(s, "version", SLA_FORMAT_VERSION);
 		XmlUtils.a_v_b(s, "bigendian", isBigEndian());
 		XmlUtils.a_v_i(s, "align", alignment);
 		XmlUtils.a_v_u(s, "uniqbase", getUniqueBase());

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/slgh_compile/SleighCompile.java

@@ -36,6 +36,7 @@ import ghidra.pcodeCPort.slghsymbol.*;
 import ghidra.pcodeCPort.space.*;
 import ghidra.pcodeCPort.utils.Utils;
 import ghidra.pcodeCPort.xml.DocumentStorage;
+import ghidra.program.model.lang.BasicCompilerSpec;
 import ghidra.sleigh.grammar.Location;
 import ghidra.util.Msg;
 
@@ -273,10 +274,15 @@ public class SleighCompile extends SleighBase {
 		// Some predefined symbols
 		root = new SubtableSymbol(location, "instruction"); // Base constructors
 		symtab.addSymbol(root);
-		insertSpace(new ConstantSpace(this, "const", 0));
+		insertSpace(new ConstantSpace(this, "const", BasicCompilerSpec.CONSTANT_SPACE_INDEX));
 		SpaceSymbol spacesym = new SpaceSymbol(location, getConstantSpace()); // Constant
 		// space
 		symtab.addSymbol(spacesym);
+		OtherSpace otherSpace = new OtherSpace(this, BasicCompilerSpec.OTHER_SPACE_NAME,
+			BasicCompilerSpec.OTHER_SPACE_INDEX);
+		insertSpace(otherSpace);
+		spacesym = new SpaceSymbol(location, otherSpace);
+		symtab.addSymbol(spacesym);
 		insertSpace(new UniqueSpace(this, "unique", numSpaces(), 0));
 		spacesym = new SpaceSymbol(location, getUniqueSpace()); // Temporary register
 		// space

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/space/AddrSpace.java

@@ -19,6 +19,11 @@ import java.io.PrintStream;
 import java.util.StringTokenizer;
 
 import org.jdom.Element;
+
+import ghidra.pcodeCPort.error.LowlevelError;
+import ghidra.pcodeCPort.pcoderaw.VarnodeData;
+import ghidra.pcodeCPort.translate.Translate;
+import ghidra.pcodeCPort.utils.*;
 /// \brief A region where processor data is stored
 ///
 /// An AddrSpace (Address Space) is an arbitrary sequence of
@@ -59,9 +64,18 @@ public class AddrSpace {
 
 	public static final AddrSpace MIN_SPACE = new AddrSpace("MIN_SPACE", -1);
 	public static final AddrSpace MAX_SPACE = new AddrSpace("MAX_SPACE", Integer.MAX_VALUE);
-	protected static final int big_endian = 1;
-	protected static final int heritaged = 2;
-	public static final int hasphysical = 4;
+
+	//see space.hh
+	protected static final int big_endian = 1;		       // Space is big endian if set, little endian otherwise
+	protected static final int heritaged = 2;		       // This space is heritaged
+	protected static final int does_deadcode = 4;		   // Dead-code analysis is done on this space
+	protected static final int programspecific = 8;        // Space is specific to a particular loadimage
+	protected static final int reverse_justification = 16; // Justification within aligned word is opposite of endianness
+	protected static final int overlay = 32;		       // This space is an overlay of another space
+	protected static final int overlaybase = 64;		   // This is the base space for overlay space(s)
+	protected static final int truncated = 128;		       // Space is truncated from its original size, expect pointers larger than this size
+	public static final int hasphysical = 256;		       // Has physical memory associated with it
+	protected static final int is_otherspace = 512;  	   // Quick check for OtherSpace
 
 	private int flags;
 	private long highest;
@@ -188,6 +202,10 @@ public class AddrSpace {
 		return ((flags & big_endian) != 0);
 	}
 
+	public boolean isOtherSpace() {
+		return ((flags & is_otherspace) != 0);
+	}
+
 	public AddrSpace getContain() {
 		return null;
 	}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/space/OtherSpace.java

@@ -0,0 +1,49 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.pcodeCPort.space;
+
+import java.io.PrintStream;
+
+import ghidra.pcodeCPort.translate.Translate;
+
+public class OtherSpace extends AddrSpace {
+
+	public OtherSpace(Translate t, String nm, int ind) {
+		super(t, spacetype.IPTR_PROCESSOR, nm, 8, 1, ind, 0, 0);
+		clearFlags(heritaged);
+		setFlags(is_otherspace);
+	}
+
+	public OtherSpace(Translate t) {
+		super(t, spacetype.IPTR_PROCESSOR);
+		clearFlags(heritaged);
+		setFlags(is_otherspace);
+	}
+
+	@Override
+	public int printRaw(PrintStream s, long offset) {
+		s.append("0x");
+		s.append(Long.toHexString(offset));
+		return getTrans().getDefaultSize();
+	}
+
+	@Override
+	public void saveXml(PrintStream s) {
+		s.print("<space_other");
+		save_basic_attributes(s);
+		s.println("/>");
+	}
+}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/pcodeCPort/translate/Translate.java

@@ -28,6 +28,7 @@ import ghidra.pcodeCPort.pcoderaw.VarnodeData;
 import ghidra.pcodeCPort.space.*;
 import ghidra.pcodeCPort.utils.AddrSpaceToIdSymmetryMap;
 import ghidra.pcodeCPort.xml.DocumentStorage;
+import ghidra.program.model.lang.BasicCompilerSpec;
 
 public abstract class Translate implements BasicSpaceProvider {
 
@@ -245,6 +246,9 @@ public abstract class Translate implements BasicSpaceProvider {
 		else if ("space_unique".equals(tp)) {
 			res = new UniqueSpace(this);
 		}
+		else if ("space_other".equals(tp)) {
+			res = new OtherSpace(this);
+		}
 		else {
 			res = new AddrSpace(this, spacetype.IPTR_PROCESSOR);
 		}
@@ -255,7 +259,11 @@ public abstract class Translate implements BasicSpaceProvider {
 
 	protected void restoreXmlSpaces(Element el) {
 		// The first space should always be the constant space
-		insertSpace(new ConstantSpace(this, "const", 0));
+		insertSpace(new ConstantSpace(this, "const", BasicCompilerSpec.CONSTANT_SPACE_INDEX));
+
+		// The second space should always be the other space
+		insertSpace(new OtherSpace(this, BasicCompilerSpec.OTHER_SPACE_NAME,
+			BasicCompilerSpec.OTHER_SPACE_INDEX));
 
 		String defname = el.getAttributeValue("defaultspace");
 		List<?> children = el.getChildren();
@@ -399,6 +407,11 @@ public abstract class Translate implements BasicSpaceProvider {
 				}
 				// fallthru
 			case IPTR_PROCESSOR:
+				if (spc.isOtherSpace()) {
+					if (spc.getIndex() != BasicCompilerSpec.OTHER_SPACE_INDEX) {
+						throw new LowlevelError("OTHER space must be assigned index 1");
+					}
+				}
 				for (AddrSpace space : baselist) {
 					if (space.getName().equals(spc.getName())) {
 						duplicatedefine = true;

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramAddressFactory.java

@@ -101,11 +101,13 @@ public class ProgramAddressFactory extends DefaultAddressFactory {
 		addAddressSpace(ovSpace);
 	}
 
-	public OverlayAddressSpace addOverlayAddressSpace(String name,AddressSpace originalSpace,
+	public OverlayAddressSpace addOverlayAddressSpace(String name, AddressSpace originalSpace,
 			long minOffset, long maxOffset) throws DuplicateNameException {
 		int unique = 0;
-		if (originalSpace.getType() == AddressSpace.TYPE_RAM)
+		if (originalSpace.getType() == AddressSpace.TYPE_RAM ||
+			originalSpace.getType() == AddressSpace.TYPE_OTHER) {
 			unique = getNextUniqueID();
+		}
 		OverlayAddressSpace ovSpace =
 			new OverlayAddressSpace(name, originalSpace, unique, minOffset, maxOffset);
 		addAddressSpace(ovSpace);
@@ -145,15 +147,16 @@ public class ProgramAddressFactory extends DefaultAddressFactory {
 	}
 
 	@Override
-	protected void renameOverlaySpace(String oldName, String newName) throws DuplicateNameException {
+	protected void renameOverlaySpace(String oldName, String newName)
+			throws DuplicateNameException {
 		super.renameOverlaySpace(oldName, newName);
 	}
 
 	private int getNextUniqueID() {
 		int maxID = 0;
 		AddressSpace[] spaces = getAllAddressSpaces();
-		for (int i = 0; i < spaces.length; i++) {
-			maxID = Math.max(maxID, spaces[i].getUnique());
+		for (AddressSpace space : spaces) {
+			maxID = Math.max(maxID, space.getUnique());
 		}
 		return maxID + 1;
 	}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDB.java

@@ -1776,6 +1776,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
 			finally {
 				program.setEventsEnabled(true);
 			}
+
 			fireBlockRemoved(startAddress);
 			if (startAddress.getAddressSpace().isOverlaySpace()) {
 				checkRemoveAddressSpace(startAddress.getAddressSpace());

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/address/AddressSpace.java

@@ -17,6 +17,7 @@ package ghidra.program.model.address;
 
 import java.math.BigInteger;
 
+import ghidra.program.model.lang.BasicCompilerSpec;
 import ghidra.program.model.listing.Program;
 
 /**
@@ -66,39 +67,39 @@ public interface AddressSpace extends Comparable<AddressSpace> {
 
 	/**
 	 * The <code>OTHER_SPACE</code> is used to store data from the original program file that doesn't
-	 * get loaded into the final memory image.
+	 * get loaded into the final memory image and for user-defined spaces.
 	 */
-	public static final AddressSpace OTHER_SPACE = new GenericAddressSpace("OTHER", 64, TYPE_OTHER,
-		0);
+	public static final AddressSpace OTHER_SPACE = new GenericAddressSpace(
+		BasicCompilerSpec.OTHER_SPACE_NAME, 64, TYPE_OTHER, BasicCompilerSpec.OTHER_SPACE_INDEX);
 
 	/**
 	 * The <code>EXTERNAL_SPACE</code> is used to contain all external locations (i.e., data and functions) 
 	 * defined within a given library namespace.  All external locations within a program
 	 * are given a unique offset within the EXTERNAL space.
 	 */
-	public static final AddressSpace EXTERNAL_SPACE = new GenericAddressSpace("EXTERNAL", 32,
-		TYPE_EXTERNAL, 0);
+	public static final AddressSpace EXTERNAL_SPACE =
+		new GenericAddressSpace("EXTERNAL", 32, TYPE_EXTERNAL, 0);
 
 	/**
 	 * The <code>VARIABLE_SPACE</code> is used to contain all variables and parameters 
 	 * defined within a given namespace (i.e., function).  All variables within a program
 	 * are given a unique offset within the VARIABLE space.
 	 */
-	public static final AddressSpace VARIABLE_SPACE = new GenericAddressSpace("VARIABLE", 32,
-		TYPE_VARIABLE, 0);
+	public static final AddressSpace VARIABLE_SPACE =
+		new GenericAddressSpace("VARIABLE", 32, TYPE_VARIABLE, 0);
 
 	/**
 	 * The <code>HASH_SPACE</code> provides a 60-bit space for encoding of unique hashcodes. 
 	 */
-	public static final AddressSpace HASH_SPACE = new GenericAddressSpace("HASH", 60, TYPE_UNKNOWN,
-		0);
+	public static final AddressSpace HASH_SPACE =
+		new GenericAddressSpace("HASH", 60, TYPE_UNKNOWN, 0);
 
 	/**
 	 * A language may only define a single REGISTER space.  If one is not defined, this 
 	 * DEFAULT_REGISTER_SPACE definition will be used.
 	 */
-	public static final AddressSpace DEFAULT_REGISTER_SPACE = new GenericAddressSpace("REGISTER",
-		32, TYPE_REGISTER, 0);
+	public static final AddressSpace DEFAULT_REGISTER_SPACE =
+		new GenericAddressSpace("REGISTER", 32, TYPE_REGISTER, 0);
 
 	/**
 	 * Returns the name of this address space.

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/lang/BasicCompilerSpec.java

@@ -46,6 +46,11 @@ public class BasicCompilerSpec implements CompilerSpec {
 
 	public static final String STACK_SPACE_NAME = "stack";
 	public static final String JOIN_SPACE_NAME = "join";
+	public static final String OTHER_SPACE_NAME = "OTHER";
+
+	//must match AddrSpace enum (see space.hh)
+	public static final int CONSTANT_SPACE_INDEX = 0;
+	public static final int OTHER_SPACE_INDEX = 1;
 
 	private final CompilerSpecDescription description;
 	private String sourceName;
@@ -117,8 +122,9 @@ public class BasicCompilerSpec implements CompilerSpec {
 			parseException = e;
 			Throwable cause = e.getCause();		// Recover the cause (from the validator exception)
 			if (cause != null) {
-				if (cause instanceof SAXException || cause instanceof IOException)
+				if (cause instanceof SAXException || cause instanceof IOException) {
 					parseException = (Exception) cause;
+				}
 			}
 		}
 		catch (FileNotFoundException e) {
@@ -361,6 +367,9 @@ public class BasicCompilerSpec implements CompilerSpec {
 		else {
 			space = language.getAddressFactory().getAddressSpace(spaceName);
 		}
+		if (spaceName.equals(OTHER_SPACE_NAME)) {
+			space = AddressSpace.OTHER_SPACE;
+		}
 		if (space == null) {
 			throw new SleighException("Unknown address space: " + spaceName);
 		}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/pcode/Varnode.java

@@ -406,8 +406,13 @@ public class Varnode {
 	public static void appendSpaceOffset(StringBuilder buf, Address addr) {
 		AddressSpace space = addr.getAddressSpace();
 		if (space.isOverlaySpace()) {
-			space = space.getPhysicalSpace();
-			addr = space.getAddress(addr.getOffset());
+			//if the address is contained within the overlay space, stay in the overlay
+			//otherwise default to the underlying space
+			if (addr.compareTo(space.getMinAddress()) < 0 ||
+				addr.compareTo(space.getMaxAddress()) > 0) {
+				space = space.getPhysicalSpace();
+				addr = space.getAddress(addr.getOffset());
+			}
 		}
 		SpecXmlUtils.encodeStringAttribute(buf, "space", space.getName());
 		SpecXmlUtils.encodeUnsignedIntegerAttribute(buf, "offset", addr.getUnsignedOffset());