GP-2677: Introduce TraceRmi (API only, experimental)

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/pcode/exec/trace/data/AbstractPcodeTraceDataAccess.java

@@ -108,7 +108,12 @@ public abstract class AbstractPcodeTraceDataAccess implements InternalPcodeTrace
 		if (hostRange == null) {
 			return;
 		}
-		getMemoryOps(true).setState(snap, toOverlay(hostRange), state);
+		TraceMemoryOperations ops = getMemoryOps(true);
+		if (ops == null) {
+			throw new AssertionError("Cannot get memory operations for writing. " +
+				"This usually indicates a schema issue.");
+		}
+		ops.setState(snap, toOverlay(hostRange), state);
 	}
 
 	@Override
@@ -178,7 +183,12 @@ public abstract class AbstractPcodeTraceDataAccess implements InternalPcodeTrace
 		if (hostStart == null) {
 			return 0;
 		}
-		return getMemoryOps(true).putBytes(snap, toOverlay(hostStart), buf);
+		TraceMemoryOperations ops = getMemoryOps(true);
+		if (ops == null) {
+			throw new AssertionError("Cannot get memory operations for writing. " +
+				"This usually indicates a schema issue.");
+		}
+		return ops.putBytes(snap, toOverlay(hostStart), buf);
 	}
 
 	@Override

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemoryBufferEntry.java

@@ -131,9 +131,16 @@ public class DBTraceMemoryBufferEntry extends DBAnnotatedObject {
 		if (compressed) {
 			decompress();
 		}
-		buffer.put((blockNum << DBTraceMemorySpace.BLOCK_SHIFT) + dstOffset, buf.array(),
-			buf.arrayOffset() + buf.position(), len);
-		buf.position(buf.position() + len);
+		int bufOffset = (blockNum << DBTraceMemorySpace.BLOCK_SHIFT) + dstOffset;
+		if (buf.isReadOnly()) {
+			byte[] temp = new byte[len];
+			buf.get(temp);
+			buffer.put(bufOffset, temp);
+		}
+		else {
+			buffer.put(bufOffset, buf.array(), buf.arrayOffset() + buf.position(), len);
+			buf.position(buf.position() + len);
+		}
 		return len;
 	}
 

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/memory/DBTraceMemorySpace.java

@@ -645,7 +645,7 @@ public class DBTraceMemorySpace
 	@Override
 	public int putBytes(long snap, Address start, ByteBuffer buf) {
 		assertInSpace(start);
-		int arrOff = buf.arrayOffset() + buf.position();
+		int pos = buf.position();
 		try (LockHold hold = LockHold.lock(lock.writeLock())) {
 
 			ByteBuffer oldBytes = ByteBuffer.allocate(buf.remaining());
@@ -659,7 +659,8 @@ public class DBTraceMemorySpace
 				doSetState(snap, start, end, TraceMemoryState.KNOWN);
 
 				// Read back the written bytes and fire event
-				byte[] bytes = Arrays.copyOfRange(buf.array(), arrOff, arrOff + result);
+				byte[] bytes = new byte[result];
+				buf.get(pos, bytes);
 				ImmutableTraceAddressSnapRange tasr = new ImmutableTraceAddressSnapRange(start,
 					start.add(result - 1), snap, lastSnap.snap);
 				trace.setChanged(new TraceChangeRecord<>(TraceMemoryBytesChangeType.CHANGED,

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/space/AbstractDBTraceSpaceBasedManager.java

@@ -95,6 +95,10 @@ public abstract class AbstractDBTraceSpaceBasedManager<M extends DBTraceSpaceBas
 			return space.isRegisterSpace();
 		}
 
+		private boolean isOverlaySpace() {
+			return space.isOverlaySpace();
+		}
+
 		private Frame frame() {
 			return new Frame(thread, entry.frameLevel);
 		}
@@ -145,7 +149,7 @@ public abstract class AbstractDBTraceSpaceBasedManager<M extends DBTraceSpaceBas
 		Map<Frame, TabledSpace> newRegSpaces = new HashMap<>();
 		Map<AddressSpace, TabledSpace> newMemSpaces = new HashMap<>();
 		for (TabledSpace ts : getTabledSpaces()) {
-			if (ts.isRegisterSpace()) {
+			if (ts.isRegisterSpace() && !ts.isOverlaySpace()) {
 				newRegSpaces.put(ts.frame(), ts);
 			}
 			else {

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/target/DBTraceObject.java

@@ -223,6 +223,8 @@ public class DBTraceObject extends DBAnnotatedObject implements TraceObject {
 		}
 		DBTraceObject parent = doCreateCanonicalParentObject();
 		InternalTraceObjectValue value = parent.setValue(lifespan, path.key(), this, resolution);
+		// TODO: Should I re-order the recursion, so values are inserted from root to this?
+		// TODO: Should child lifespans be allowed to exceed the parent's?
 		DBTraceObjectValPath path = parent.doInsert(lifespan, resolution);
 		return path.append(value);
 	}
@@ -505,12 +507,39 @@ public class DBTraceObject extends DBAnnotatedObject implements TraceObject {
 		}
 	}
 
+	protected Lifespan doAdjust(Lifespan span, String key, Object value) {
+		// Ordered by min, so I only need to consider the first conflict
+		// If start is contained in an entry, assume the user means to overwrite it.
+		for (InternalTraceObjectValue val : doGetValues(span, key)) {
+			if (Objects.equals(value, val.getValue())) {
+				continue; // not a conflict
+			}
+			if (val.getLifespan().contains(span.min())) {
+				continue; // user probably wants to overwrite the remainder of this entry
+			}
+			// Every entry intersects the span, so if we get one, adjust
+			return span.withMax(val.getMinSnap() - 1);
+		}
+		return span;
+	}
+
 	// TODO: Could/should this return Stream instead?
 	protected Collection<? extends InternalTraceObjectValue> doGetValues(Lifespan span,
 			String key) {
 		return doGetValues(span.lmin(), span.lmax(), key);
 	}
 
+	/**
+	 * The implementation of {@link #getValues(Lifespan, String)}
+	 * 
+	 * <p>
+	 * This collects entries ordered by min snap
+	 * 
+	 * @param lower the lower snap
+	 * @param upper the upper snap
+	 * @param key the key
+	 * @return the collection of values
+	 */
 	protected Collection<? extends InternalTraceObjectValue> doGetValues(long lower, long upper,
 			String key) {
 		// Collect triplet-indexed values
@@ -746,6 +775,9 @@ public class DBTraceObject extends DBAnnotatedObject implements TraceObject {
 			if (resolution == ConflictResolution.DENY) {
 				doCheckConflicts(lifespan, key, value);
 			}
+			else if (resolution == ConflictResolution.ADJUST) {
+				lifespan = doAdjust(lifespan, key, value);
+			}
 			var setter = new ValueLifespanSetter(lifespan, value) {
 				DBTraceObject canonicalLifeChanged = null;
 

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/target/DBTraceObjectManager.java

@@ -23,6 +23,7 @@ import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.apache.commons.collections4.IteratorUtils;
 import org.jdom.JDOMException;
 
 import db.*;
@@ -192,7 +193,18 @@ public class DBTraceObjectManager implements TraceObjectManager, DBTraceManager
 			valueStore.getIndex(DBTraceObject.class, DBTraceObjectValue.CHILD_COLUMN);
 
 		objectsView = Collections.unmodifiableCollection(objectStore.asMap().values());
-		valuesView = Collections.unmodifiableCollection(valueStore.asMap().values());
+		valuesView = new AbstractCollection<>() {
+			@Override
+			public Iterator<TraceObjectValue> iterator() {
+				return IteratorUtils.chainedIterator(valueStore.asMap().values().iterator(),
+					rangeValueMap.values().iterator());
+			}
+
+			@Override
+			public int size() {
+				return objectStore.getRecordCount() + rangeValueMap.size();
+			}
+		};
 	}
 
 	protected void loadRootSchema() {

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/database/target/InternalTraceObjectValue.java

@@ -135,6 +135,9 @@ interface InternalTraceObjectValue extends TraceObjectValue {
 			if (resolution == ConflictResolution.DENY) {
 				getParent().doCheckConflicts(lifespan, getEntryKey(), getValue());
 			}
+			else if (resolution == ConflictResolution.ADJUST) {
+				lifespan = getParent().doAdjust(lifespan, getEntryKey(), getValue());
+			}
 			new ValueLifespanSetter(lifespan, getValue(), this) {
 				@Override
 				protected Iterable<InternalTraceObjectValue> getIntersecting(Long lower,
@@ -151,7 +154,8 @@ interface InternalTraceObjectValue extends TraceObjectValue {
 			}.set(lifespan, getValue());
 			if (isObject()) {
 				DBTraceObject child = getChild();
-				child.emitEvents(new TraceChangeRecord<>(TraceObjectChangeType.LIFE_CHANGED, null, child));
+				child.emitEvents(
+					new TraceChangeRecord<>(TraceObjectChangeType.LIFE_CHANGED, null, child));
 			}
 		}
 	}

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/listing/TraceDefinedDataView.java

@@ -21,7 +21,6 @@ import ghidra.program.model.lang.Register;
 import ghidra.program.model.util.CodeUnitInsertionException;
 import ghidra.trace.model.Lifespan;
 import ghidra.trace.model.guest.TracePlatform;
-import ghidra.trace.util.TraceRegisterUtils;
 
 /**
  * A view of defined data units
@@ -74,8 +73,8 @@ public interface TraceDefinedDataView extends TraceBaseDefinedUnitsView<TraceDat
 	 */
 	default TraceData create(Lifespan lifespan, Register register, DataType dataType)
 			throws CodeUnitInsertionException {
-		TraceRegisterUtils.requireByteBound(register);
-		return create(lifespan, register.getAddress(), dataType, register.getNumBytes());
+		return create(getTrace().getPlatformManager().getHostPlatform(), lifespan, register,
+			dataType);
 	}
 
 	/**

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/target/TraceObject.java

@@ -47,6 +47,13 @@ public interface TraceObject extends TraceUniqueObject {
 	 */
 	Trace getTrace();
 
+	/**
+	 * Get the database key for this object
+	 * 
+	 * @return the key
+	 */
+	long getKey();
+
 	/**
 	 * Get the root of the tree containing this object
 	 * 
@@ -171,7 +178,11 @@ public interface TraceObject extends TraceUniqueObject {
 		 * Throw {@link DuplicateKeyException} if the specified lifespan would result in conflicting
 		 * entries
 		 */
-		DENY;
+		DENY,
+		/**
+		 * Adjust the new entry to fit into the span available, possibly ignoring it altogether
+		 */
+		ADJUST;
 	}
 
 	/**

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/target/TraceObjectInterface.java

@@ -48,11 +48,19 @@ public interface TraceObjectInterface {
 
 	@Transitional
 	default long computeMinSnap() {
-		return computeSpan().lmin();
+		Lifespan span = computeSpan();
+		if (span == null) {
+			return 0;
+		}
+		return span.lmin();
 	}
 
 	@Transitional
 	default long computeMaxSnap() {
-		return computeSpan().lmax();
+		Lifespan span = computeSpan();
+		if (span == null) {
+			return 0;
+		}
+		return span.lmax();
 	}
 }

Ghidra/Debug/Framework-TraceModeling/src/main/java/ghidra/trace/model/target/TraceObjectValue.java

@@ -61,6 +61,17 @@ public interface TraceObjectValue {
 	 */
 	Object getValue();
 
+	/**
+	 * A convenience to get and cast the value, without checking
+	 * 
+	 * @param <T> the desired type
+	 * @return the value
+	 */
+	@SuppressWarnings("unchecked")
+	default <T> T castValue() {
+		return (T) getValue();
+	}
+
 	/**
 	 * Get the value as an object
 	 * 

Ghidra/Debug/Framework-TraceModeling/src/test/java/ghidra/trace/database/ToyDBTraceBuilder.java

@@ -26,11 +26,13 @@ import java.nio.charset.CharsetEncoder;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
+import java.util.List;
 
-import db.Transaction;
 import db.DBHandle;
+import db.Transaction;
 import generic.theme.GThemeDefaults.Colors.Messages;
 import ghidra.app.plugin.processors.sleigh.SleighLanguage;
+import ghidra.dbg.util.PathPredicates;
 import ghidra.pcode.exec.*;
 import ghidra.pcode.exec.trace.TraceSleighUtils;
 import ghidra.program.disassemble.Disassembler;
@@ -52,6 +54,7 @@ import ghidra.trace.model.*;
 import ghidra.trace.model.guest.TraceGuestPlatform;
 import ghidra.trace.model.guest.TracePlatform;
 import ghidra.trace.model.symbol.TraceReferenceManager;
+import ghidra.trace.model.target.*;
 import ghidra.trace.model.thread.TraceThread;
 import ghidra.util.Msg;
 import ghidra.util.database.DBOpenMode;
@@ -750,6 +753,60 @@ public class ToyDBTraceBuilder implements AutoCloseable {
 		return getLanguage(langID).getCompilerSpecByID(new CompilerSpecID(compID));
 	}
 
+	/**
+	 * Get an object by its canonical path
+	 * 
+	 * @param canonicalPath the canonical path
+	 * @return the object or null
+	 */
+	public TraceObject obj(String canonicalPath) {
+		return trace.getObjectManager()
+				.getObjectByCanonicalPath(TraceObjectKeyPath.parse(canonicalPath));
+	}
+
+	
+	/**
+	 * Get an object by its path pattern
+	 * 
+	 * @param path the path pattern
+	 * @return the object or null
+	 */
+	public TraceObject objAny(String pat) {
+		return objAny(pat, Lifespan.at(0));
+	}
+	public TraceObject objAny(String path, Lifespan span) {
+		return trace.getObjectManager().getObjectsByPath(span, TraceObjectKeyPath.parse(path))
+				.findFirst()
+				.orElse(null);
+	}
+
+	/**
+	 * Get the value (not value entry) of an object
+	 * 
+	 * @param obj the object
+	 * @param snap the snapshot key
+	 * @param key the entry key
+	 * @return the value, possibly null
+	 */
+	public Object objValue(TraceObject obj, long snap, String key) {
+		TraceObjectValue value = obj.getValue(snap, key);
+		return value == null ? null : value.getValue();
+	}
+
+	/**
+	 * List all values matching the given pattern at the given stnap.
+	 * 
+	 * @param snap the snap
+	 * @param pattern the pattern
+	 * @return the list of values
+	 */
+	public List<Object> objValues(long snap, String pattern) {
+		return trace.getObjectManager()
+				.getValuePaths(Lifespan.at(snap), PathPredicates.parse(pattern))
+				.map(p -> p.getDestinationValue(trace.getObjectManager().getRootObject()))
+				.toList();
+	}
+
 	@Override
 	public void close() {
 		if (trace.getConsumerList().contains(this)) {