GP-4252 handle bulk datatype replacements more efficiently

This commit is contained in:
ghidra1 2024-01-22 12:08:35 -05:00
parent e15e12b248
commit 9dce76ae53
11 changed files with 128 additions and 113 deletions

View file

@ -16,7 +16,8 @@
package ghidra.trace.database.data;
import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import db.DBHandle;
@ -184,16 +185,14 @@ public class DBTraceDataTypeManager extends ProgramBasedDataTypeManagerDB
}
@Override
protected void replaceDataTypeIDs(long oldID, long newID) {
if (oldID == newID) {
return;
}
trace.getCodeManager().replaceDataTypes(oldID, newID);
trace.getSymbolManager().replaceDataTypes(oldID, newID);
protected void replaceDataTypesUsed(Map<Long, Long> dataTypeReplacementMap) {
trace.getCodeManager().replaceDataTypes(dataTypeReplacementMap);
trace.getSymbolManager().replaceDataTypes(dataTypeReplacementMap);
}
@Override
protected void deleteDataTypeIDs(LinkedList<Long> deletedIds) {
protected void deleteDataTypesUsed(Set<Long> deletedIds) {
// TODO: Should use replacement type instead of clearing
trace.getCodeManager().clearData(deletedIds, TaskMonitor.DUMMY);
trace.getSymbolManager().invalidateCache(false);
}

View file

@ -433,8 +433,7 @@ public class DBTraceCodeManager extends AbstractDBTraceSpaceBasedManager<DBTrace
}
@Override
public DBTraceCodeSpace getCodeRegisterSpace(TraceThread thread,
boolean createIfAbsent) {
public DBTraceCodeSpace getCodeRegisterSpace(TraceThread thread, boolean createIfAbsent) {
return getForRegisterSpace(thread, 0, createIfAbsent);
}
@ -445,26 +444,24 @@ public class DBTraceCodeManager extends AbstractDBTraceSpaceBasedManager<DBTrace
}
@Override
public DBTraceCodeSpace getCodeRegisterSpace(TraceStackFrame frame,
boolean createIfAbsent) {
public DBTraceCodeSpace getCodeRegisterSpace(TraceStackFrame frame, boolean createIfAbsent) {
return getForRegisterSpace(frame, createIfAbsent);
}
@Internal
public void replaceDataTypes(long oldID, long newID) {
public void replaceDataTypes(Map<Long, Long> dataTypeReplacementMap) {
TODO();
}
@Internal
public void clearData(LinkedList<Long> deletedDataTypeIds, TaskMonitor monitor) {
public void clearData(Set<Long> deletedDataTypeIds, TaskMonitor monitor) {
TODO();
}
@Internal
public void clearPlatform(Lifespan span, AddressRange range, DBTraceGuestPlatform guest,
TaskMonitor monitor) throws CancelledException {
delegateDeleteV(range.getAddressSpace(),
m -> m.clearPlatform(span, range, guest, monitor));
delegateDeleteV(range.getAddressSpace(), m -> m.clearPlatform(span, range, guest, monitor));
}
@Internal

View file

@ -138,10 +138,7 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
@DBAnnotatedColumn(STORAGE_COLUMN_NAME)
static DBObjectColumn STORAGE_COLUMN;
@DBAnnotatedField(
column = STORAGE_COLUMN_NAME,
indexed = true,
codec = VariableStorageDBFieldCodec.class)
@DBAnnotatedField(column = STORAGE_COLUMN_NAME, indexed = true, codec = VariableStorageDBFieldCodec.class)
private VariableStorage storage;
protected final DBTraceSymbolManager manager;
@ -241,8 +238,7 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
public DBTraceSymbolManager(DBHandle dbh, DBOpenMode openMode, ReadWriteLock lock,
TaskMonitor monitor, Language baseLanguage, DBTrace trace,
DBTraceThreadManager threadManager, DBTraceDataTypeManager dataTypeManager,
DBTraceOverlaySpaceAdapter overlayAdapter)
throws VersionException, IOException {
DBTraceOverlaySpaceAdapter overlayAdapter) throws VersionException, IOException {
this.trace = trace;
this.lock = lock;
this.threadManager = threadManager;
@ -278,8 +274,7 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
allNamespaces = new DBTraceSymbolMultipleTypesView<>(this, namespaces, classes);
uniqueNamespaces =
new DBTraceSymbolMultipleTypesNoDuplicatesView<>(this, namespaces, classes);
notLabels =
new DBTraceSymbolMultipleTypesNoDuplicatesView<>(this, namespaces, classes);
notLabels = new DBTraceSymbolMultipleTypesNoDuplicatesView<>(this, namespaces, classes);
allSymbols = new DBTraceSymbolMultipleTypesView<>(this, labels, namespaces, classes);
}
@ -357,7 +352,7 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
}
// Internal
public void replaceDataTypes(long oldID, long newID) {
public void replaceDataTypes(Map<Long, Long> dataTypeReplacementMap) {
// Would apply to functions and variables, but those are not supported.
}
@ -521,9 +516,8 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
delID(thread, symbol.getAddress().getAddressSpace(), symbol.getID());
}
// TODO: Remove from other space maps, once implemented.
trace.setChanged(
new TraceChangeRecord<>(TraceSymbolChangeType.DELETED, symbol.getSpace(), symbol, null,
null));
trace.setChanged(new TraceChangeRecord<>(TraceSymbolChangeType.DELETED, symbol.getSpace(),
symbol, null, null));
return true;
}
@ -593,8 +587,8 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
Collection<Long> result = new ArrayList<>();
for (DBTraceAddressSnapRangePropertyMapSpace<Long, DBTraceSymbolIDEntry> space : idMap
.getActiveMemorySpaces()) {
result.addAll(space
.reduce(TraceAddressSnapRangeQuery.added(from, to, space.getAddressSpace()))
result.addAll(
space.reduce(TraceAddressSnapRangeQuery.added(from, to, space.getAddressSpace()))
.values());
}
return result;
@ -608,8 +602,8 @@ public class DBTraceSymbolManager implements TraceSymbolManager, DBTraceManager
Collection<Long> result = new ArrayList<>();
for (DBTraceAddressSnapRangePropertyMapSpace<Long, DBTraceSymbolIDEntry> space : idMap
.getActiveMemorySpaces()) {
result.addAll(space
.reduce(TraceAddressSnapRangeQuery.removed(from, to, space.getAddressSpace()))
result.addAll(
space.reduce(TraceAddressSnapRangeQuery.removed(from, to, space.getAddressSpace()))
.values());
}
return result;

View file

@ -104,9 +104,15 @@ public class DeleteAction extends DockingAction {
@Override
public void actionPerformed(ActionContext context) {
//@formatter:off
int choice = OptionDialog.showYesNoDialogWithNoAsDefaultButton(null,
"Confirm Delete Operation", "Are you sure you want to delete selected\n categories " +
"and/or dataTypes?\n(Note: There is no undo for archives.)");
"Confirm Delete Operation",
"Are you sure you want to delete selected\n" +
"data types and/or categories?\n\n" +
"Note: There is no undo for archives and\n" +
"changes may trigger the removal of related\n" +
"data types, components and defined data.)");
//@formatter:on
if (choice != OptionDialog.OPTION_ONE) {
return;
}

View file

@ -16,12 +16,13 @@
package ghidra.program.database;
import java.io.IOException;
import java.util.LinkedList;
import java.util.*;
import javax.help.UnsupportedOperationException;
import db.*;
import db.util.ErrorHandler;
import generic.stl.Pair;
import ghidra.framework.model.DomainFile;
import ghidra.framework.store.LockException;
import ghidra.program.model.data.*;
@ -104,7 +105,6 @@ public class ProjectDataTypeManager extends StandAloneDataTypeManager
super.setProgramArchitecture(language, compilerSpecId, updateOption, monitor);
}
////////////////////
@Override
public void dataTypeChanged(DataType dt, boolean isAutoChange) {
super.dataTypeChanged(dt, isAutoChange);
@ -184,14 +184,13 @@ public class ProjectDataTypeManager extends StandAloneDataTypeManager
super.favoritesChanged(dataType, isFavorite);
}
///////////////////
@Override
protected void replaceDataTypeIDs(long oldDataTypeID, long newDataTypeID) {
protected void replaceDataTypesUsed(Map<Long, Long> dataTypeReplacementMap) {
// do nothing
}
@Override
protected void deleteDataTypeIDs(LinkedList<Long> deletedIds) {
protected void deleteDataTypesUsed(Set<Long> deletedIds) {
// do nothing
}

View file

@ -2673,11 +2673,11 @@ public class CodeManager implements ErrorHandler, ManagerDB {
/**
* Removes any data objects that have dataTypes matching the given dataType ids.
* @param dataTypeIDs the list of ids of dataTypes that have been deleted.
* @param dataTypeIDs the set of {@link DataType} IDs that have been deleted.
* @param monitor the task monitor.
* @throws CancelledException if cancelled
*/
public void clearData(long[] dataTypeIDs, TaskMonitor monitor) throws CancelledException {
public void clearData(Set<Long> dataTypeIDs, TaskMonitor monitor) throws CancelledException {
lock.acquire();
try {
List<Address> addrs = new ArrayList<>();
@ -3434,15 +3434,16 @@ public class CodeManager implements ErrorHandler, ManagerDB {
return "";
}
public void replaceDataTypes(long oldDataTypeID, long newDataTypeID) {
public void replaceDataTypes(Map<Long, Long> dataTypeReplacementMap) {
lock.acquire();
try {
RecordIterator it = dataAdapter.getRecords();
while (it.hasNext()) {
DBRecord rec = it.next();
long id = rec.getLongValue(DataDBAdapter.DATA_TYPE_ID_COL);
if (id == oldDataTypeID) {
rec.setLongValue(DataDBAdapter.DATA_TYPE_ID_COL, newDataTypeID);
Long replacementId = dataTypeReplacementMap.get(id);
if (replacementId != null) {
rec.setLongValue(DataDBAdapter.DATA_TYPE_ID_COL, replacementId);
dataAdapter.putRecord(rec);
Address addr = addrMap.decodeAddress(rec.getKey());
program.setChanged(ProgramEvent.CODE_REPLACED, addr, addr, null, null);

View file

@ -154,6 +154,10 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
private LinkedList<Long> idsToDelete = new LinkedList<>();
private LinkedList<Pair<DataType, DataType>> typesToReplace = new LinkedList<>();
private List<DataType> favoritesList = new ArrayList<>();
// TODO: idsToDataTypeMap may have issue since there could be a one to many mapping
// (e.g., type with same UniversalID could be in multiple categories unless specifically
// prevented during resolve)
private IdsToDataTypeMap idsToDataTypeMap = new IdsToDataTypeMap();
private ThreadLocal<EquivalenceCache> equivalenceCache = new ThreadLocal<>();
@ -1907,14 +1911,35 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
private void replaceQueuedDataTypes() {
// collect all datatypes to be replaced and notify children which may also get queued
// Collect all datatypes to be replaced and notify children which may also get queued
// for removal.
LinkedList<Pair<DataType, DataType>> dataTypeReplacements = new LinkedList<>();
Map<Long, Long> dataTypeReplacementMap = new HashMap<>();
List<Pair<DataType, DataType>> dataTypeReplacements = new LinkedList<>();
while (!typesToReplace.isEmpty()) {
Pair<DataType, DataType> dataTypeReplacement = typesToReplace.removeFirst();
replaceUsesInOtherDataTypes(dataTypeReplacement.first, dataTypeReplacement.second);
dataTypeReplacements.addFirst(dataTypeReplacement);
DataType replacedDt = dataTypeReplacement.first;
DataType replacementDt = dataTypeReplacement.second;
long replacedId = getID(replacedDt);
if (replacedId <= 0) {
Msg.error(this, "Unexpected ID for replaced datatype (" + replacedId + "): " +
replacedDt.getPathName());
continue; // ignore attempt to replace special type
}
long replacementId = getID(dataTypeReplacement.second);
if (replacementId < 0) {
Msg.error(this, "Unexpected ID for replacement datatype (" + replacementId + "): " +
replacementDt.getPathName());
}
replaceUsesInOtherDataTypes(replacedDt, replacementDt);
dataTypeReplacements.add(dataTypeReplacement);
dataTypeReplacementMap.put(replacedId, replacementId);
}
// perform any neccessary external use replacements
replaceDataTypesUsed(dataTypeReplacementMap);
// perform actual database updates (e.g., record updates, change notifications, etc.)
for (Pair<DataType, DataType> dataTypeReplacement : dataTypeReplacements) {
@ -1922,16 +1947,21 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
}
}
private void replaceDataType(DataType existingDt, DataType replacementDt) {
/**
* Allow extensions to perform any neccessary fixups to address all datatype replacements.
* @param dataTypeReplacementMap map of datatype replacements (oldID maps to replacementID).
*/
abstract protected void replaceDataTypesUsed(Map<Long, Long> dataTypeReplacementMap);
DataTypePath replacedDtPath = existingDt.getDataTypePath();
long replacedId = getID(existingDt);
private void replaceDataType(DataType replacedDt, DataType replacementDt) {
UniversalID id = existingDt.getUniversalID();
idsToDataTypeMap.removeDataType(existingDt.getSourceArchive(), id);
DataTypePath replacedDtPath = replacedDt.getDataTypePath();
long replacedId = getID(replacedDt);
UniversalID id = replacedDt.getUniversalID();
idsToDataTypeMap.removeDataType(replacedDt.getSourceArchive(), id);
try {
replaceDataTypeIDs(replacedId, getID(replacementDt));
parentChildAdapter.removeAllRecordsForParent(replacedId);
}
catch (IOException e) {
@ -1964,13 +1994,6 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
}
}
/**
* Replace all datatype uses external to the datatype manager if applicable.
* @param oldID old datatype ID
* @param newID new datatype ID
*/
abstract protected void replaceDataTypeIDs(long oldID, long newID);
/**
* Replace one source archive (oldDTM) with another (newDTM). Any data types
* whose source was the oldDTM will be changed to have a source that is the
@ -2177,12 +2200,12 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
if (dt == DataType.DEFAULT) {
return DEFAULT_DATATYPE_ID;
}
if (dt instanceof BitFieldDataType) {
return createKey(BITFIELD, BitFieldDBDataType.getId((BitFieldDataType) dt));
}
if (dt instanceof BadDataType) {
return BAD_DATATYPE_ID;
}
if (dt instanceof BitFieldDataType) {
return createKey(BITFIELD, BitFieldDBDataType.getId((BitFieldDataType) dt));
}
if (dt instanceof DataTypeDB) {
// NOTE: Implementation DOES NOT check or guarantee that datatype or its returned ID
// correspond to this datatype manager instance. This seems incorrect although it's
@ -2235,40 +2258,48 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
* Remove the given datatype from this manager (assumes the lock has already been acquired).
*
* @param dataType the dataType to be removed
* @return true if datatype removal was successful, else false
*/
private void removeInternal(DataType dataType) {
private boolean removeInternal(DataType dataType) {
if (!contains(dataType)) {
return;
return false;
}
long id = getID(dataType);
if (id < 0) {
return;
if (id <= 0) { // removal of certain special types not permitted
return false;
}
idsToDelete.add(Long.valueOf(id));
removeQueuedDataTypes();
return true;
}
private void removeQueuedDataTypes() {
// collect all datatype to be removed and notify children which may also get queued
// for removal.
LinkedList<Long> deletedIds = new LinkedList<>();
Set<Long> deletedIds = new HashSet<>();
while (!idsToDelete.isEmpty()) {
long id = idsToDelete.removeFirst();
removeUseOfDataType(id);
deletedIds.addFirst(id);
deletedIds.add(id);
}
// perform any neccessary external use removals
deleteDataTypesUsed(deletedIds);
// perform actual database updates (e.g., record removal, change notifications, etc.)
for (long id : deletedIds) {
deleteDataType(id);
}
// Remove all uses of datatypes external to datatype manager
deleteDataTypeIDs(deletedIds);
}
/**
* Allow extensions to perform any neccessary fixups for all datatype removals listed.
* @param deletedIds list of IDs for all datatypes which are getting removed.
*/
protected abstract void deleteDataTypesUsed(Set<Long> deletedIds);
private void removeUseOfDataType(long id) {
if (isBulkRemoving) {
@ -2292,8 +2323,7 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
lock.acquire();
try {
if (contains(dataType)) {
removeInternal(dataType);
return true;
return removeInternal(dataType);
}
}
finally {
@ -2404,12 +2434,6 @@ abstract public class DataTypeManagerDB implements DataTypeManager {
typesToReplace.add(new Pair<>(oldDataType, replacementDataType));
}
/**
* Delete all datatype uses external to the datatype manager if applicable.
* @param deletedIds old datatype IDs which were deleted
*/
abstract protected void deleteDataTypeIDs(LinkedList<Long> deletedIds);
private void notifyDeleted(long dataTypeID) {
DataType dataType = getDataType(dataTypeID);
if (dataType == null) {

View file

@ -16,7 +16,8 @@
package ghidra.program.database.data;
import java.io.IOException;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import db.*;
import db.util.ErrorHandler;
@ -241,31 +242,25 @@ public class ProgramDataTypeManager extends ProgramBasedDataTypeManagerDB implem
}
@Override
protected void replaceDataTypeIDs(long oldDataTypeID, long newDataTypeID) {
if (oldDataTypeID == newDataTypeID) {
return;
}
program.getCodeManager().replaceDataTypes(oldDataTypeID, newDataTypeID);
program.getSymbolTable().replaceDataTypes(oldDataTypeID, newDataTypeID);
program.getFunctionManager().replaceDataTypes(oldDataTypeID, newDataTypeID);
protected void replaceDataTypesUsed(Map<Long, Long> dataTypeReplacementMap) {
program.getCodeManager().replaceDataTypes(dataTypeReplacementMap);
program.getSymbolTable().replaceDataTypes(dataTypeReplacementMap);
program.getFunctionManager().replaceDataTypes(dataTypeReplacementMap);
}
@Override
protected void deleteDataTypeIDs(LinkedList<Long> deletedIds) {
protected void deleteDataTypesUsed(Set<Long> deletedIds) {
// TODO: SymbolManager/FunctionManager do not appear to handle datatype removal update.
// Suspect it handles indirectly through detection of deleted datatype. Old deleted ID
// use could be an issue.
long[] ids = new long[deletedIds.size()];
int i = 0;
for (Long deletedId : deletedIds) {
ids[i++] = deletedId.longValue();
}
try {
program.getCodeManager().clearData(ids, TaskMonitor.DUMMY);
// TODO: Should use replacement type instead of clearing
program.getCodeManager().clearData(deletedIds, TaskMonitor.DUMMY);
}
catch (CancelledException e) {
// won't happen
}
program.getSymbolTable().invalidateCache(false);
program.getFunctionManager().invalidateCache(false);
}

View file

@ -1202,7 +1202,7 @@ public class FunctionManagerDB implements FunctionManager {
}
}
public void replaceDataTypes(long oldDataTypeID, long newDataTypeID) {
public void replaceDataTypes(Map<Long, Long> dataTypeReplacementMap) {
lock.acquire();
try {
RecordIterator it = adapter.iterateFunctionRecords();
@ -1213,8 +1213,10 @@ public class FunctionManagerDB implements FunctionManager {
continue; // skip thunks
}
if (rec.getLongValue(FunctionAdapter.RETURN_DATA_TYPE_ID_COL) == oldDataTypeID) {
rec.setLongValue(FunctionAdapter.RETURN_DATA_TYPE_ID_COL, newDataTypeID);
long id = rec.getLongValue(FunctionAdapter.RETURN_DATA_TYPE_ID_COL);
Long replacementId = dataTypeReplacementMap.get(id);
if (replacementId != null) {
rec.setLongValue(FunctionAdapter.RETURN_DATA_TYPE_ID_COL, replacementId);
adapter.updateFunctionRecord(rec);
FunctionDB functionDB = cache.get(rec);
if (functionDB == null) {

View file

@ -672,9 +672,7 @@ public class SymbolManager implements SymbolTable, ManagerDB {
while (symIt.hasNext()) {
list.add(symIt.next());
}
Iterator<Symbol> it = list.iterator();
while (it.hasNext()) {
Symbol s = it.next();
for (Symbol s : list) {
s.delete();
}
}
@ -1524,9 +1522,7 @@ public class SymbolManager implements SymbolTable, ManagerDB {
DBRecord rec = iter.next();
symbols.add(getSymbol(rec));
}
Iterator<SymbolDB> it = symbols.iterator();
while (it.hasNext()) {
SymbolDB s = it.next();
for (SymbolDB s : symbols) {
s.delete();
}
}
@ -2209,7 +2205,7 @@ public class SymbolManager implements SymbolTable, ManagerDB {
variableStorageMgr.setLanguage(translator, monitor);
}
public void replaceDataTypes(long oldDataTypeID, long newDataTypeID) {
public void replaceDataTypes(Map<Long, Long> dataTypeReplacementMap) {
lock.acquire();
try {
RecordIterator it = adapter.getSymbols();
@ -2238,8 +2234,9 @@ public class SymbolManager implements SymbolTable, ManagerDB {
}
}
long id = rec.getLongValue(SymbolDatabaseAdapter.SYMBOL_DATATYPE_COL);
if (id == oldDataTypeID) {
rec.setLongValue(SymbolDatabaseAdapter.SYMBOL_DATATYPE_COL, newDataTypeID);
Long replacementId = dataTypeReplacementMap.get(id);
if (replacementId != null) {
rec.setLongValue(SymbolDatabaseAdapter.SYMBOL_DATATYPE_COL, replacementId);
adapter.updateSymbolRecord(rec);
symbolDataChanged(getSymbol(rec));
}

View file

@ -17,7 +17,7 @@ package ghidra.program.model.data;
import java.io.Closeable;
import java.io.IOException;
import java.util.LinkedList;
import java.util.*;
import javax.help.UnsupportedOperationException;
@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList;
import db.*;
import db.util.ErrorHandler;
import generic.jar.ResourceFile;
import generic.stl.Pair;
import ghidra.framework.model.RuntimeIOException;
import ghidra.framework.store.LockException;
import ghidra.program.database.DBStringMapAdapter;
@ -870,12 +871,12 @@ public class StandAloneDataTypeManager extends DataTypeManagerDB implements Clos
}
@Override
protected void replaceDataTypeIDs(long oldID, long newID) {
protected void replaceDataTypesUsed(Map<Long, Long> dataTypeReplacementMap) {
// do nothing
}
@Override
protected void deleteDataTypeIDs(LinkedList<Long> deletedIds) {
protected void deleteDataTypesUsed(Set<Long> deletedIds) {
// do nothing
}