Merge remote-tracking branch 'origin/Ghidra_11.3'

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/assembler/sleigh/AbstractSleighAssemblerBuilder.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -86,10 +86,6 @@ public abstract class AbstractSleighAssemblerBuilder< //
 	 * @throws SleighException if there's an issue accessing the language
 	 */
 	protected void generateAssembler() throws SleighException {
-		if (generated) {
-			return;
-		}
-		generated = true;
 		try {
 			buildGrammar();
 			grammar.verify();
@@ -106,15 +102,23 @@ public abstract class AbstractSleighAssemblerBuilder< //
 		}
 	}
 
+	private void checkGenerateAssembler() throws SleighException {
+		if (generated) {
+			return;
+		}
+		generated = true;
+		generateAssembler();
+	}
+
 	@Override
 	public A getAssembler(AssemblySelector selector) {
-		generateAssembler();
+		checkGenerateAssembler();
 		return newAssembler(selector);
 	}
 
 	@Override
 	public A getAssembler(AssemblySelector selector, Program program) {
-		generateAssembler();
+		checkGenerateAssembler();
 		return newAssembler(selector, program);
 	}
 

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/assembler/sleigh/sem/AssemblyPatternBlock.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,12 +17,14 @@ package ghidra.app.plugin.assembler.sleigh.sem;
 
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicLong;
 
 import ghidra.app.plugin.assembler.sleigh.expr.MaskedLong;
 import ghidra.app.plugin.assembler.sleigh.expr.SolverException;
 import ghidra.app.plugin.assembler.sleigh.util.AsmUtil;
+import ghidra.app.plugin.processors.sleigh.ContextCommit;
 import ghidra.app.plugin.processors.sleigh.ContextOp;
 import ghidra.app.plugin.processors.sleigh.expression.ContextField;
 import ghidra.app.plugin.processors.sleigh.expression.TokenField;
@@ -403,6 +405,60 @@ public class AssemblyPatternBlock implements Comparable<AssemblyPatternBlock> {
 		return new AssemblyPatternBlock(newOffset, newMask, newVals);
 	}
 
+	/**
+	 * Combine this pattern block with another given block
+	 * 
+	 * <p>
+	 * The two blocks are combined regardless if their corresponding defined bits agree. When blocks
+	 * are combined, their bytes are aligned according to their shifts, and the defined bits are
+	 * taken from either block. If neither block defines a bit (i.e., the mask bit at that position
+	 * is 0 for both input blocks), then the output has an undefined bit in the corresponding
+	 * position. If both blocks define the bit, but they have opposite values, then the value from
+	 * <code>that</code> takes precedence.
+	 * 
+	 * @see RegisterValue#combineValues(RegisterValue)
+	 * 
+	 * @param that the other block
+	 * @return the new combined block
+	 */
+	public AssemblyPatternBlock assign(AssemblyPatternBlock that) {
+		int newOffset = Math.min(this.offset, that.offset);
+		int bufLen = Math.max(this.length(), that.length()) - newOffset;
+
+		byte[] newMask = new byte[bufLen];
+		byte[] newVals = new byte[bufLen];
+
+		int diff = this.offset - newOffset;
+		for (int i = 0; i < this.mask.length; i++) {
+			newMask[diff + i] = this.mask[i];
+			newVals[diff + i] = this.vals[i];
+		}
+		diff = that.offset - newOffset;
+		for (int i = 0; i < that.mask.length; i++) {
+			byte mask = that.mask[i];
+			byte clearMask = (byte) ~mask;
+			newMask[diff + i] |= mask;
+			newVals[diff + i] = (byte) ((that.vals[i] & mask) | (newVals[diff + i] & clearMask));
+		}
+		return new AssemblyPatternBlock(newOffset, newMask, newVals);
+	}
+
+	/**
+	 * Invert the mask bits of this pattern block
+	 * 
+	 * @return a copy of this pattern block with mask bits inverted
+	 */
+	public AssemblyPatternBlock invertMask() {
+		int maskLen = this.mask.length;
+		byte[] newMask = new byte[maskLen];
+
+		for (int i = 0; i < maskLen; i++) {
+			newMask[i] = (byte) ~this.mask[i];
+		}
+
+		return new AssemblyPatternBlock(this.offset, newMask, this.vals);
+	}
+
 	@Override
 	public String toString() {
 		StringBuilder sb = new StringBuilder();
@@ -575,6 +631,30 @@ public class AssemblyPatternBlock implements Comparable<AssemblyPatternBlock> {
 		return MaskedLong.fromMaskAndValue(rmsk >>> cop.getShift(), rval >>> cop.getShift());
 	}
 
+	/**
+	 * Write mask bits from context commit to mask array of block
+	 * 
+	 * @implNote This is used when scraping for valid input contexts to determine which context variables
+	 *           are passed to the <code>globalset</code> directive.
+	 * 
+	 * @param cc the context commit
+	 * @return the result
+	 */
+	public AssemblyPatternBlock writeContextCommitMask(ContextCommit cc) {
+		byte[] newMask = Arrays.copyOf(this.mask, this.mask.length);
+		int idx = cc.getWordIndex();
+		int imsk = cc.getMask();
+		for (int i = 3; i >= 0; i--) {
+			int index = idx * 4 + i - this.offset;
+
+			if (index < newMask.length && index >= 0) {
+				newMask[index] |= imsk;
+			}
+			imsk >>= 8;
+		}
+		return new AssemblyPatternBlock(this.offset, newMask, this.vals);
+	}
+
 	/**
 	 * Set all bits read by a given context operation to unknown
 	 * 

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/assembler/sleigh/sem/AssemblyResolvedPatterns.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,6 +37,14 @@ public interface AssemblyResolvedPatterns extends AssemblyResolution {
 	 */
 	AssemblyPatternBlock getContext();
 
+	/**
+	 * Create a copy of this resolution with a new context
+	 * 
+	 * @param ctx the new context
+	 * @return the copy
+	 */
+	AssemblyResolvedPatterns withContext(AssemblyPatternBlock ctx);
+
 	/**
 	 * Get the length of the instruction encoding
 	 * 

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/assembler/sleigh/sem/DefaultAssemblyResolvedPatterns.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -560,6 +560,25 @@ public class DefaultAssemblyResolvedPatterns extends AbstractAssemblyResolution
 		return ctx;
 	}
 
+	protected AbstractAssemblyResolvedPatternsBuilder<?> withContextBuilder(
+			AssemblyPatternBlock ctx) {
+		var builder = factory.newPatternsBuilder();
+		builder.description = description;
+		builder.cons = cons;
+		builder.children = children;
+		builder.right = right;
+		builder.ins = ins;
+		builder.ctx = ctx;
+		builder.backfills = backfills;
+		builder.forbids = forbids;
+		return builder;
+	}
+
+	@Override
+	public AssemblyResolvedPatterns withContext(AssemblyPatternBlock ctx) {
+		return withContextBuilder(ctx).build();
+	}
+
 	@Override
 	public MaskedLong readInstruction(int start, int len) {
 		return ins.readBytes(start, len);

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/languages/sleigh/InputContextScraper.java

@@ -0,0 +1,129 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.app.plugin.languages.sleigh;
+
+import java.util.*;
+
+import ghidra.app.plugin.assembler.sleigh.sem.AssemblyDefaultContext;
+import ghidra.app.plugin.assembler.sleigh.sem.AssemblyPatternBlock;
+import ghidra.app.plugin.processors.sleigh.*;
+import ghidra.app.plugin.processors.sleigh.pattern.DisjointPattern;
+import ghidra.app.plugin.processors.sleigh.symbol.SubtableSymbol;
+
+/**
+ * A class for scraping input contexts from a SLEIGH language to get all of the valid input contexts
+ * that affect constructor selection
+ * 
+ */
+public class InputContextScraper {
+	private final SleighLanguage language;
+
+	public InputContextScraper(SleighLanguage language) {
+		this.language = language;
+	}
+
+	/**
+	 * Get set of all valid input contexts that affect constructor selection.
+	 * 
+	 * <ol>
+	 * <li>Start with mask of the language's default context
+	 * <li>Scrape language for <code>globalset</code> context variables and OR their masks into our
+	 * mask
+	 * <li>Flip bits of our mask to get mask of context variables not used as input
+	 * (local/transient)
+	 * <li>Check constructor constraints and use mask to get values of relevant input context
+	 * variables
+	 * </ol>
+	 */
+	public Set<AssemblyPatternBlock> scrapeInputContexts() {
+		// We don't care about the actual default values, just if a context variable HAS a default
+		// value. It's possible for a local context variable to be set in the default context, but
+		// doing so is questionable. It could be an input context variable in that case, so to
+		// account for it, we start with the default context mask. Doing so ensures those variables
+		// are included
+		AssemblyPatternBlock defaultCtx = new AssemblyDefaultContext(language).getDefault();
+
+		// Erase the values for posterity; we don't care about them at this point
+		Arrays.fill(defaultCtx.getVals(), (byte) 0);
+
+		GlobalSetScraper globalSetScraper = new GlobalSetScraper(defaultCtx);
+		SleighLanguages.traverseConstructors(language, globalSetScraper);
+
+		AssemblyPatternBlock nonInputCtxMask = globalSetScraper.getContextMask().invertMask();
+
+		ConstraintScraper constraintScraper =
+			new ConstraintScraper(nonInputCtxMask, language.getContextBaseRegister().getNumBytes());
+		SleighLanguages.traverseConstructors(language, constraintScraper);
+
+		return constraintScraper.getInputContexts();
+	}
+
+	private static class GlobalSetScraper implements ConstructorEntryVisitor {
+		private AssemblyPatternBlock contextMask;
+
+		GlobalSetScraper(AssemblyPatternBlock contextMask) {
+			this.contextMask = contextMask;
+		}
+
+		public AssemblyPatternBlock getContextMask() {
+			return contextMask;
+		}
+
+		@Override
+		public int visit(SubtableSymbol subtable, DisjointPattern pattern, Constructor cons) {
+			for (ContextChange chg : cons.getContextChanges()) {
+				if (chg instanceof ContextCommit cc) {
+					contextMask = contextMask.writeContextCommitMask(cc);
+				}
+			}
+			return CONTINUE;
+		}
+	}
+
+	private static class ConstraintScraper implements ConstructorEntryVisitor {
+		private final AssemblyPatternBlock nonInputMask;
+		private final AssemblyPatternBlock blankContext;
+		private final Set<AssemblyPatternBlock> inputContexts;
+
+		ConstraintScraper(AssemblyPatternBlock mask, int contextRegLen) {
+			nonInputMask = mask;
+			blankContext = AssemblyPatternBlock.fromLength(contextRegLen);
+			inputContexts = new HashSet<>();
+		}
+
+		public Set<AssemblyPatternBlock> getInputContexts() {
+			return inputContexts;
+		}
+
+		@Override
+		public int visit(SubtableSymbol subtable, DisjointPattern pattern, Constructor cons) {
+			AssemblyPatternBlock contextConstraint =
+				AssemblyPatternBlock.fromPattern(pattern, pattern.getLength(true), true);
+
+			if (contextConstraint.getMask().length > 0) {
+				// Combine constraint with blank context to ensure generated context has no shifts
+				AssemblyPatternBlock inputCtx =
+					blankContext.combine(contextConstraint).maskOut(nonInputMask);
+
+				// Filter out entirely undefined context
+				if (inputCtx.getSpecificity() > 0) {
+					inputContexts.add(inputCtx);
+				}
+			}
+			return CONTINUE;
+		}
+	}
+}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/app/plugin/processors/sleigh/ContextCommit.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -51,4 +51,11 @@ public class ContextCommit implements ContextChange {
 		decoder.closeElement(el);
 	}
 
+	public int getWordIndex() {
+		return num;
+	}
+
+	public int getMask() {
+		return mask;
+	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramContentHandler.java

@@ -28,6 +28,7 @@ import ghidra.framework.model.ChangeSet;
 import ghidra.framework.model.DomainObject;
 import ghidra.framework.store.*;
 import ghidra.framework.store.local.LocalDatabaseItem;
+import ghidra.program.model.lang.LanguageNotFoundException;
 import ghidra.util.InvalidNameException;
 import ghidra.util.Msg;
 import ghidra.util.exception.CancelledException;
@@ -232,9 +233,25 @@ public class ProgramContentHandler extends DBWithUserDataContentHandler<ProgramD
 		DBHandle userDbh =
 			openAssociatedUserFile(programItem.getFileID(), PROGRAM_CONTENT_TYPE, userfs, monitor);
 		if (userDbh != null) {
-			return new ProgramUserDataDB(userDbh, program, monitor);
+			boolean success = false;
+			try {
+				ProgramUserDataDB data = new ProgramUserDataDB(userDbh, program, monitor);
+				success = true;
+				return data;
+			}
+			catch (LanguageNotFoundException | IllegalStateException e) {
+				// Ignore - delete to make way for new one
+			}
+			finally {
+				if (!success) {
+					userDbh.close();
+					Msg.debug(this, "Removing incompatible program user data file for " +
+						programItem.getPathName());
+					removeUserDataFile(programItem, userfs);
+				}
+			}
 		}
-		return new ProgramUserDataDB(program);
+		return null; // will be created by ProgramDB when modified
 	}
 
 	private void recoverChangeSet(ProgramDB program, DBHandle dbh) throws IOException {

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramDB.java

@@ -1910,8 +1910,12 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 	}
 
 	@Override
-	protected void setChanged(boolean b) {
-		super.setChanged(b);
+	protected void setChanged(boolean state) {
+		super.setChanged(state);
+		if (!state && !dbh.isChanged()) {
+			// language upgrade has already been completed
+			languageUpgradeTranslator = null;
+		}
 	}
 
 	void setChangeSet(ProgramDBChangeSet changeSet) {
@@ -2369,6 +2373,12 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 
 	@Override
 	protected void close() {
+		if (changed && languageUpgradeTranslator != null) {
+			// Prevent user data from being saved if program and user data 
+			// have gone through a major language upgrade and the program
+			// was not saved.
+			programUserData.setChanged(false);
+		}
 		super.close();
 		intRangePropertyMap.clear();
 		addrSetPropertyMap.clear();

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramUserDataDB.java

@@ -119,6 +119,11 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 		return program.getName() + "_UserData";
 	}
 
+	/**
+	 * Create a new program user data store.
+	 * @param program related program
+	 * @throws IOException if an IO error occurs
+	 */
 	public ProgramUserDataDB(ProgramDB program) throws IOException {
 		super(new DBHandle(), getName(program), 500, program);
 		this.program = program;
@@ -157,8 +162,22 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 		}
 	}
 
+	/**
+	 * Open existing program user data store.
+	 * If a major language change is detected the instance will automatically attempt to upgrade
+	 * its internal address map.
+	 * @param dbh user data storage DB handle
+	 * @param program related program
+	 * @param monitor task monitor
+	 * @throws IOException if an IO error occurs
+	 * @throws VersionException if a DB version error occurs
+	 * @throws LanguageNotFoundException if language was not found
+	 * @throws CancelledException if instantiation was cancelled
+	 * @throws IllegalStateException if data store is bad or incmopatible with program
+	 */
 	public ProgramUserDataDB(DBHandle dbh, ProgramDB program, TaskMonitor monitor)
-			throws IOException, VersionException, LanguageNotFoundException, CancelledException {
+			throws IOException, VersionException, LanguageNotFoundException, CancelledException,
+			IllegalStateException {
 
 		super(dbh, getName(program), 500, program);
 		this.program = program;
@@ -200,6 +219,9 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 			upgradeDatabase();
 
 			if (languageVersionExc != null) {
+				if (languageUpgradeTranslator == null) {
+					throw new LanguageNotFoundException(languageID + ":" + languageVersion);
+				}
 				try {
 					setLanguage(languageUpgradeTranslator, monitor);
 					addressMap.memoryMapChanged(program.getMemory());
@@ -212,6 +234,16 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 				}
 			}
 
+			if (!program.getLanguageID().equals(languageID)) {
+				throw new IllegalStateException(
+					"User data and program have inconsistent language ID");
+			}
+
+			if (program.getLanguage().getVersion() != languageVersion) {
+				throw new IllegalStateException(
+					"User data language version does not match program's");
+			}
+
 			endTransaction(id, true);
 			changed = false;
 			clearUndo(false);
@@ -431,6 +463,10 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 			record.setString(VALUE_COL, languageID.getIdAsString());
 			table.putRecord(record);
 
+			record = SCHEMA.createRecord(new StringField(LANGUAGE_VERSION));
+			record.setString(VALUE_COL, Integer.toString(languageVersion));
+			table.putRecord(record);
+
 			setChanged(true);
 			clearCache(true);
 
@@ -451,6 +487,11 @@ class ProgramUserDataDB extends DomainObjectAdapterDB implements ProgramUserData
 		return dbh.canUpdate();
 	}
 
+	@Override
+	protected void setChanged(boolean b) {
+		super.setChanged(b);
+	}
+	
 	private PropertyMap<?> getPropertyMap(String owner, String propertyName, int propertyType,
 			Class<?> saveableClass, boolean create) throws PropertyTypeMismatchException {