GP-3765 Updated AutoVersionTracking script and task to process implied matches when running from the script if user wants them.

Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java

@@ -23,6 +23,7 @@ import ghidra.feature.vt.api.main.VTSession;
 import ghidra.feature.vt.api.util.VTOptions;
 import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask;
 import ghidra.feature.vt.gui.plugin.VTController;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.model.DomainFolder;
 import ghidra.framework.options.ToolOptions;
 import ghidra.framework.plugintool.PluginTool;
@@ -68,6 +69,9 @@ public class AutoVersionTrackingScript extends GhidraScript {
 			return;
 		}
 
+		boolean autoCreateImpliedMatches = askYesNo("Implied Matches?",
+			"Would you like the script to figure out implied matches from any matches it creates?");
+
 		// Need to end the script transaction or it interferes with vt things that need locks
 		end(true);
 
@@ -76,12 +80,32 @@ public class AutoVersionTrackingScript extends GhidraScript {
 
 		folder.createFile(name, session, monitor);
 
+		ToolOptions options = getOptions();
+
+		boolean originalImpliedMatchSetting =
+			options.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+		options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, autoCreateImpliedMatches);
+
 		AutoVersionTrackingTask autoVtTask =
-			new AutoVersionTrackingTask(session, getOptions(), 1.0, 10.0);
+			new AutoVersionTrackingTask(session, options, 0.95, 10.0);
+
 
 		TaskLauncher.launch(autoVtTask);
+
+
+		// if not running headless user can decide whether to save or not
+		// if running headless - must save here or nothing that was done in this script will be
+		// accessible later.
+		if (isRunningHeadless()) {
+			session.save();
+		}
+		options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, originalImpliedMatchSetting);
+
+		println(autoVtTask.getStatusMsg());
 	}
 
+
 	private ToolOptions getOptions() {
 		ToolOptions vtOptions = new VTOptions("Dummy");
 		PluginTool tool = state.getTool();

Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java

@@ -15,28 +15,61 @@
  */
 package ghidra.feature.vt.gui.actions;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
 
 import javax.swing.SwingConstants;
 
-import ghidra.feature.vt.api.correlator.program.*;
+import ghidra.feature.vt.api.correlator.program.CombinedFunctionAndDataReferenceProgramCorrelatorFactory;
-import ghidra.feature.vt.api.main.*;
+import ghidra.feature.vt.api.correlator.program.DataReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.DuplicateFunctionMatchProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactDataMatchProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchBytesProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchInstructionsProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.ExactMatchMnemonicsProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.FunctionReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.SymbolNameProgramCorrelatorFactory;
+import ghidra.feature.vt.api.correlator.program.VTAbstractReferenceProgramCorrelatorFactory;
+import ghidra.feature.vt.api.main.VTAssociation;
+import ghidra.feature.vt.api.main.VTAssociationManager;
+import ghidra.feature.vt.api.main.VTAssociationStatus;
+import ghidra.feature.vt.api.main.VTAssociationType;
+import ghidra.feature.vt.api.main.VTMarkupItem;
+import ghidra.feature.vt.api.main.VTMatch;
+import ghidra.feature.vt.api.main.VTMatchSet;
+import ghidra.feature.vt.api.main.VTProgramCorrelator;
+import ghidra.feature.vt.api.main.VTProgramCorrelatorFactory;
+import ghidra.feature.vt.api.main.VTSession;
 import ghidra.feature.vt.api.util.VTAssociationStatusException;
 import ghidra.feature.vt.api.util.VTOptions;
 import ghidra.feature.vt.gui.plugin.AddressCorrelatorManager;
 import ghidra.feature.vt.gui.task.ApplyMarkupItemTask;
+import ghidra.feature.vt.gui.util.ImpliedMatchUtils;
 import ghidra.feature.vt.gui.util.MatchInfo;
 import ghidra.feature.vt.gui.util.MatchInfoFactory;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.options.ToolOptions;
 import ghidra.program.model.address.Address;
 import ghidra.program.model.address.AddressSetView;
 import ghidra.program.model.lang.OperandType;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.CodeUnit;
+import ghidra.program.model.listing.CodeUnitIterator;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionManager;
+import ghidra.program.model.listing.Instruction;
+import ghidra.program.model.listing.InstructionIterator;
+import ghidra.program.model.listing.Program;
 import ghidra.program.util.ListingDiff;
 import ghidra.util.Msg;
 import ghidra.util.exception.AssertException;
 import ghidra.util.exception.CancelledException;
-import ghidra.util.task.*;
+import ghidra.util.task.Task;
+import ghidra.util.task.TaskMonitor;
+import ghidra.util.task.WrappingTaskMonitor;
 import util.CollectionUtils;
 
 /**
@@ -142,9 +175,24 @@ public class AutoVersionTrackingTask extends Task {
 		int count = 0;
 		monitor.doInitialize(NUM_CORRELATORS);
 
+		// save user option and use to determine whether to handle implied matches at all later
+		boolean autoCreateImpliedMatches =
+			applyOptions.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+		// Turn off auto implied matches and handle later if user had that option set
+		// This is because when run from the VT GUI action implied matches are created automatically
+		// by the VT controller when the option is set but they are not created when called from a 
+		// script since there is no VT controller in that case. If allowed to happen in 
+		// GUI then they will happen twice when called later in this task and the implied match 
+		// votes will be wrong. This Task doesn't know if called from GUI or script so this is 
+		// klunky but will make sure they are only processed once and will make sure the user option
+		// is put back the way the user had it. 
+		applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
 		// Use default options for all of the "exact" correlators; passed in options for the others
 		VTOptions options;
 
+
 		// Run the correlators in the following order:
 		// Do this one first because we don't want it to find ones that get markup applied by later
 		// correlators
@@ -266,6 +314,63 @@ public class AutoVersionTrackingTask extends Task {
 				" with some apply markup errors. See the log or the markup table for more details";
 		}
 		statusMsg = NAME + " completed successfully" + applyMarkupStatus;
+
+		// if user had implied match option chosen then figure out implied matches now
+		if (autoCreateImpliedMatches) {
+			processImpliedMatches(monitor);
+		}
+
+		// reset auto implied match option to user choice
+		applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH,
+			autoCreateImpliedMatches);
+
+	}
+
+	private void processImpliedMatches(TaskMonitor monitor) throws CancelledException {
+
+		List<VTAssociation> processedSrcDestPairs = new ArrayList<>();
+		List<VTMatchSet> matchSets = session.getMatchSets();
+
+		monitor.setMessage("Processing Implied Matches...");
+		monitor.initialize(matchSets.size());
+
+		for (VTMatchSet matchSet : matchSets) {
+			monitor.checkCancelled();
+
+			Collection<VTMatch> matches = matchSet.getMatches();
+			for (VTMatch match : matches) {
+				monitor.checkCancelled();
+
+				VTAssociation association = match.getAssociation();
+
+				// Implied matches currently only created for functions so skip matches that are
+				// data matches
+				if (association.getType() == VTAssociationType.DATA) {
+					continue;
+				}
+
+				// Implied matches should only be created for matches that user has accepted as 
+				// good matches
+				if (association.getStatus() != VTAssociationStatus.ACCEPTED) {
+					continue;
+				}
+				// only process the same match pair once so implied vote counts are not overinflated
+				if (processedSrcDestPairs.contains(association)) {
+					continue;
+				}
+
+				MatchInfo matchInfo = matchInfoFactory.getMatchInfo(match, addressCorrelator);
+
+				ImpliedMatchUtils.updateImpliedMatchForAcceptedAssocation(
+					matchInfo.getSourceFunction(),
+					matchInfo.getDestinationFunction(), session,
+					addressCorrelator, monitor);
+
+				processedSrcDestPairs.add(association);
+			}
+			monitor.incrementProgress();
+		}
+
 	}
 
 	private int getNumberOfDataMatches(TaskMonitor monitor) throws CancelledException {

Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java

@@ -15,27 +15,48 @@
  */
 package ghidra.feature.vt.api;
 
-import static ghidra.feature.vt.db.VTTestUtils.*;
+import static ghidra.feature.vt.db.VTTestUtils.addr;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 
-import org.junit.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
 import ghidra.app.cmd.disassemble.DisassembleCommand;
 import ghidra.feature.vt.api.db.VTSessionDB;
-import ghidra.feature.vt.api.main.*;
+import ghidra.feature.vt.api.main.VTAssociationStatus;
+import ghidra.feature.vt.api.main.VTAssociationType;
+import ghidra.feature.vt.api.main.VTMatch;
+import ghidra.feature.vt.api.main.VTMatchInfo;
+import ghidra.feature.vt.api.main.VTMatchSet;
+import ghidra.feature.vt.api.main.VTProgramCorrelator;
+import ghidra.feature.vt.api.main.VTScore;
+import ghidra.feature.vt.api.main.VTSession;
 import ghidra.feature.vt.api.util.VTAssociationStatusException;
 import ghidra.feature.vt.db.VTTestUtils;
 import ghidra.feature.vt.gui.VTTestEnv;
 import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask;
 import ghidra.feature.vt.gui.plugin.VTController;
+import ghidra.feature.vt.gui.util.VTOptionDefines;
 import ghidra.framework.options.Options;
+import ghidra.framework.options.ToolOptions;
 import ghidra.program.database.ProgramDB;
 import ghidra.program.database.function.OverlappingFunctionException;
 import ghidra.program.model.address.Address;
 import ghidra.program.model.address.AddressSet;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.CodeUnit;
+import ghidra.program.model.listing.CodeUnitIterator;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionManager;
+import ghidra.program.model.listing.Listing;
+import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.MemoryAccessException;
 import ghidra.program.model.symbol.SourceType;
 import ghidra.test.AbstractGhidraHeadedIntegrationTest;
@@ -778,6 +799,50 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe
 		}
 	}
 
+	/*
+	 * This tests auto version tracking with auto implied matches option set
+	 */
+	@Test
+	public void testRunAutoVT_impliedMatches() throws Exception {
+
+		sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME);
+		destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME);
+
+		session = env.createSession(sourceProgram, destinationProgram);
+
+		env.showTool();
+		controller = env.getVTController();
+		ToolOptions options = controller.getOptions();
+		options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, true);
+
+		// Score .999999 and confidence 10.0 (log10 confidence 2.0) and up
+		runAutoVTCommand(0.999999999, 10.0);
+
+		assertTrue(session.getImpliedMatchSet().getMatchCount() > 0);
+	}
+
+	/*
+	 * This tests auto version tracking with auto implied matches option not set
+	 */
+	@Test
+	public void testRunAutoVT_noImpliedMatches() throws Exception {
+
+		sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME);
+		destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME);
+
+		session = env.createSession(sourceProgram, destinationProgram);
+
+		env.showTool();
+		controller = env.getVTController();
+		ToolOptions options = controller.getOptions();
+		options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false);
+
+		// Score .999999 and confidence 10.0 (log10 confidence 2.0) and up
+		runAutoVTCommand(0.999999999, 10.0);
+
+		assertTrue(session.getImpliedMatchSet().getMatchCount() == 0);
+	}
+
 	private VTMatch createMatch(Address sourceAddress, Address destinationAddress,
 			boolean setAccepted) throws VTAssociationStatusException {
 		VTProgramCorrelator correlator =