From d4d703368b7589c85faead36dba25f67d1ca9702 Mon Sep 17 00:00:00 2001 From: ghidra007 Date: Mon, 20 Nov 2023 22:32:31 +0000 Subject: [PATCH] GP-3934 Added options to autoVT task and script. Made example options script for headless. Updated Documentation. Updated and added tests. --- .../VersionTracking/certification.manifest | 1 + .../AutoVersionTrackingScript.java | 257 +++++++++++-- .../SetAutoVersionTrackingOptionsScript.java | 105 ++++++ .../VersionTrackingPlugin/VT_AutoVT.html | 353 +++++++++++++++++ .../topics/VersionTrackingPlugin/VT_Tool.html | 43 +-- .../providers/VT_Apply_Options.html | 2 +- .../actions/AutoVersionTrackingAction.java | 21 +- .../gui/actions/AutoVersionTrackingTask.java | 355 ++++++++++------- .../matchtable/VTMatchTableProvider.java | 219 ++++++++++- .../feature/vt/gui/util/VTOptionDefines.java | 109 +++++- .../vt/api/VTAutoVersionTrackingTest.java | 357 +++++++++++++++--- 11 files changed, 1542 insertions(+), 280 deletions(-) create mode 100644 Ghidra/Features/VersionTracking/ghidra_scripts/SetAutoVersionTrackingOptionsScript.java create mode 100644 Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_AutoVT.html diff --git a/Ghidra/Features/VersionTracking/certification.manifest b/Ghidra/Features/VersionTracking/certification.manifest index 3cc0271000..fe6191d51e 100644 --- a/Ghidra/Features/VersionTracking/certification.manifest +++ b/Ghidra/Features/VersionTracking/certification.manifest @@ -9,6 +9,7 @@ data/version.tracking.theme.properties||GHIDRA||||END| src/main/docs/VTClasses.png||GHIDRA||reviewed||END| src/main/docs/VTGuiImpl.png||GHIDRA||reviewed||END| src/main/help/help/TOC_Source.xml||GHIDRA||reviewed||END| +src/main/help/help/topics/VersionTrackingPlugin/VT_AutoVT.html||GHIDRA||||END| src/main/help/help/topics/VersionTrackingPlugin/VT_Correlators.html||GHIDRA||||END| src/main/help/help/topics/VersionTrackingPlugin/VT_Preconditions.html||GHIDRA||||END| src/main/help/help/topics/VersionTrackingPlugin/VT_Tool.html||GHIDRA||||END| diff --git a/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java b/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java index fd7d02584d..8934e5fb00 100644 --- a/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java +++ b/Ghidra/Features/VersionTracking/ghidra_scripts/AutoVersionTrackingScript.java @@ -13,21 +13,51 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// An example of how to create Version Tracking session, run some correlators to find matching -// data and and then save the session. -//@category Examples.Version Tracking - +// A script that runs Auto Version Tracking given the options set in one of the following ways: +// 1. If script is run from the CodeBrowser, the GUI options are set in a pop up dialog by user. +// 2. If script is run in headless mode either the defaults provided by the script are used or the +// user can specify a script to be run that sets the options. See example script +// SetAutoVersionTrackingOptionsScript that can be copied and updated to reflect the users +// desired options. +// +// NOTE: This is an example to show how run this script in headless mode +// +// /support/analyzeHeadless.bat/sh c:/MyGhidraProjectFolder +// MyProjectName/OptionalFolderContainingProgram -process Program1.exe -postScript +// MyOptionsSetupScript -postScript AutoVersionTrackingScript.java "/FolderContainingSession" +// "MySessionName" true "/OptionalFolderContainingProgram/Program2.exe" +// +// +// NOTE: The first program will be analyzed for you if it is not already analyzed (and if you +// do not include the -noanalysis option) as it is part of the typical analyzeHeadless run. +// The second program must be analyzed prior to running the script as the headless analyzer +// itself knows nothing about the file other than as a given option name. This is true in +// both GUI and headless mode. +// +// NOTE: The second to last parameter is to identify whether the first listed program +// is the source program or not. True means first program is source program and second +// program is destination program. False means second program is source program and first +// program is destination program. This is important if you want the correct markup to be +// applied from the source to destination program. +// +// NOTE: The options setup script is optional. It is only necessary if users want to change the +// default options. To use it make a copy of the example one and save to a new script. You +// You may need to add the -scriptPath to the headless run so it will find your script. +// +//@category Version Tracking import ghidra.app.script.GhidraScript; import ghidra.feature.vt.api.db.VTSessionDB; import ghidra.feature.vt.api.main.VTSession; import ghidra.feature.vt.api.util.VTOptions; import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask; -import ghidra.feature.vt.gui.plugin.VTController; import ghidra.feature.vt.gui.util.VTOptionDefines; +import ghidra.features.base.values.GhidraValuesMap; +import ghidra.framework.model.DomainFile; import ghidra.framework.model.DomainFolder; import ghidra.framework.options.ToolOptions; -import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; +import ghidra.util.MessageType; +import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskLauncher; public class AutoVersionTrackingScript extends GhidraScript { @@ -49,47 +79,106 @@ public class AutoVersionTrackingScript extends GhidraScript { @Override public void run() throws Exception { - DomainFolder folder = - askProjectFolder("Please choose a folder for your Version Tracking session."); - String name = askString("Please enter a Version Tracking session name", "Session Name"); + GhidraValuesMap startupValues = new GhidraValuesMap(); + + startupValues.defineProjectFolder("Version Tracking Session Folder", "/"); + startupValues.defineString("Version Tracking Session Name"); + startupValues.defineBoolean("Check if current program is the Source Program", true); + startupValues.defineProgram("Please select the other program"); + + startupValues.setValidator((valueMap, status) -> { + + GhidraValuesMap map = (GhidraValuesMap) valueMap; + + if (!valueMap.hasValue("Version Tracking Session Name")) { + status.setStatusText("Session Name must be filled in!", MessageType.ERROR); + return false; + } + + try { + if (hasExistingSession(map.getString("Version Tracking Session Name"), + map.getProjectFolder("Version Tracking Session Folder"))) { + status.setStatusText("Session cannot be an existing session!", + MessageType.ERROR); + return false; + } + } + catch (CancelledException e) { + return false; + } + + if (!valueMap.hasValue("Please select the other program")) { + status.setStatusText("Must choose second program!", MessageType.ERROR); + return false; + } + return true; + }); + + startupValues = askValues("Enter Auto Version Tracking Information", + "Changing these options will not change the corresponding tool options", + startupValues); + + DomainFolder folder = startupValues.getProjectFolder("Version Tracking Session Folder"); + + String name = startupValues.getString("Version Tracking Session Name"); + boolean isCurrentProgramSourceProg = + startupValues.getBoolean("Check if current program is the Source Program"); + Program otherProgram = + startupValues.getProgram("Please select the other program", this, state.getTool()); - boolean isCurrentProgramSourceProg = askYesNo("Current Program Source Program?", - "Is the current program your source program?"); if (isCurrentProgramSourceProg) { sourceProgram = currentProgram; - destinationProgram = askProgram("Please select the destination (new) program"); + destinationProgram = otherProgram; } else { destinationProgram = currentProgram; - sourceProgram = askProgram("Please select the source (existing annotated) program"); + sourceProgram = otherProgram; } if (sourceProgram == null || destinationProgram == null) { return; } - boolean autoCreateImpliedMatches = askYesNo("Implied Matches?", - "Would you like the script to figure out implied matches from any matches it creates?"); // Need to end the script transaction or it interferes with vt things that need locks end(true); + VTSession session = VTSessionDB.createVTSession(name, sourceProgram, destinationProgram, this); - folder.createFile(name, session, monitor); - ToolOptions options = getOptions(); + if (folder.getFile(name) == null) { + folder.createFile(name, session, monitor); + } - boolean originalImpliedMatchSetting = - options.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + // create a default options map in case cannot get user input + GhidraValuesMap optionsMap = createDefaultOptions(); - options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, autoCreateImpliedMatches); + // if running script in GUI get options from user and update the vtOptions with them + if(!isRunningHeadless()) { + optionsMap = getOptionsFromUser(); + + } + // else if running script in headless get possible options set by prescript that saves + // optionsMap in script state variable and update the vtOptions with them + else { + // try to get options map from state if running headless + // if user runs prescript to set up their own options map those options will be used + // See SetAutoVersionTrackingOptionsScript.java as an example + GhidraValuesMap stateOptionsMap = + (GhidraValuesMap) state.getEnvironmentVar("autoVTOptionsMap"); + if (optionsMap != null) { + optionsMap = stateOptionsMap; + } + + } + + ToolOptions vtOptions = setToolOptionsFromOptionsMap(optionsMap); AutoVersionTrackingTask autoVtTask = - new AutoVersionTrackingTask(session, options, 0.95, 10.0); - + new AutoVersionTrackingTask(session, vtOptions); TaskLauncher.launch(autoVtTask); @@ -98,21 +187,131 @@ public class AutoVersionTrackingScript extends GhidraScript { // if running headless - must save here or nothing that was done in this script will be // accessible later. if (isRunningHeadless()) { + otherProgram.save("Updated with Auto Version Tracking", monitor); session.save(); } - options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, originalImpliedMatchSetting); + println(autoVtTask.getStatusMsg()); + otherProgram.release(this); } + /** + * Method to determine if there is an existing VTSession in the given folder with the given name + * @param name the given name + * @param folder the given Ghidra project folder + * @return true if there is an existing VTSession with the given name in the given folder, false + * otherwise + * @throws CancelledException if cancelled + */ + private boolean hasExistingSession(String name, DomainFolder folder) throws CancelledException { - private ToolOptions getOptions() { - ToolOptions vtOptions = new VTOptions("Dummy"); - PluginTool tool = state.getTool(); - if (tool != null) { - vtOptions = tool.getOptions(VTController.VERSION_TRACKING_OPTIONS_NAME); + DomainFile[] files = folder.getFiles(); + + for (DomainFile file : files) { + monitor.checkCancelled(); + + if (file.getName().equals(name)) { + if (file.getContentType().equals("VersionTracking")) { + return true; + } + } } - return vtOptions; + return false; + } + + /** + * Method to create the default GhidraValuesMap AutoVT options + * @return the default GhidraValuesMap AutoVT options + */ + private GhidraValuesMap createDefaultOptions() { + GhidraValuesMap optionsValues = new GhidraValuesMap(); + + optionsValues.defineBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_SYMBOL_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION_TEXT, true); + optionsValues.defineBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION_TEXT, true); + optionsValues.defineInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION_TEXT, 5); + optionsValues.defineInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION_TEXT, 3); + optionsValues.defineInt(VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT, 10); + optionsValues.defineInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT, 10); + optionsValues.defineBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION_TEXT, true); + optionsValues.defineInt(VTOptionDefines.MIN_VOTES_OPTION_TEXT, 2); + optionsValues.defineInt(VTOptionDefines.MAX_CONFLICTS_OPTION_TEXT, 0); + optionsValues.defineDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION_TEXT, 0.95); + optionsValues.defineDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION_TEXT, 10.0); + + return optionsValues; + } + + /** + * Method to ask the user for AutoVT options + * @return a GhidraValuesMap containing AutoVT options values + * @throws CancelledException if cancelled + */ + private GhidraValuesMap getOptionsFromUser() throws CancelledException { + + GhidraValuesMap optionsValues = createDefaultOptions(); + + optionsValues = askValues("Enter Auto Version Tracking Options", + "These options will not be saved to your current tool options.", + optionsValues); + + return optionsValues; + } + + /** + * Set the Auto Version Tracking options given a GhidraValuesMap containing the options values + * @param optionsValues the option values in a GhidraValuesMap + * @return ToolOptions containing the Auto Version Tracking options values + */ + private ToolOptions setToolOptionsFromOptionsMap(GhidraValuesMap optionsValues) { + + ToolOptions toolOptions = new VTOptions("Dummy"); + + toolOptions.setBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION, + optionsValues.getBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_EXACT_SYMBOL_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_EXACT_SYMBOL_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION_TEXT)); + toolOptions.setBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION, + optionsValues.getBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION_TEXT)); + + toolOptions.setInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION, + optionsValues.getInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION_TEXT)); + toolOptions.setInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION, + optionsValues.getInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION_TEXT)); + toolOptions.setInt(VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION, + optionsValues.getInt(VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT)); + toolOptions.setInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION, + optionsValues.getInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT)); + + toolOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, + optionsValues.getDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION_TEXT)); + toolOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, + optionsValues.getDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION_TEXT)); + + toolOptions.setBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION, + optionsValues.getBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION_TEXT)); + + toolOptions.setInt(VTOptionDefines.MIN_VOTES_OPTION, + optionsValues.getInt(VTOptionDefines.MIN_VOTES_OPTION_TEXT)); + + toolOptions.setInt(VTOptionDefines.MAX_CONFLICTS_OPTION, + optionsValues.getInt(VTOptionDefines.MAX_CONFLICTS_OPTION_TEXT)); + + return toolOptions; + } } diff --git a/Ghidra/Features/VersionTracking/ghidra_scripts/SetAutoVersionTrackingOptionsScript.java b/Ghidra/Features/VersionTracking/ghidra_scripts/SetAutoVersionTrackingOptionsScript.java new file mode 100644 index 0000000000..00e9d5ea0e --- /dev/null +++ b/Ghidra/Features/VersionTracking/ghidra_scripts/SetAutoVersionTrackingOptionsScript.java @@ -0,0 +1,105 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Example script to set AutoVersionTrackcing options to be used when running AutoVersionTracking +// script in headless mode. Users should copy and rename this script and update the given default +// option values to ones they prefer. +//@category Examples.Version Tracking +import ghidra.app.script.GhidraScript; +import ghidra.feature.vt.gui.util.VTOptionDefines; +import ghidra.features.base.values.GhidraValuesMap; + +public class SetAutoVersionTrackingOptionsScript extends GhidraScript { + + @Override + public void run() throws Exception { + + GhidraValuesMap optionsMap = getOptions(); + state.addEnvironmentVar("autoVTOptionsMap", optionsMap); + } + + private GhidraValuesMap getOptions() { + + GhidraValuesMap optionsValues = new GhidraValuesMap(); + + // If true, this option will cause the Auto Version Tracker to create implied matches for any + // applied function matches by other correlators + optionsValues.defineBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run the Exact Symbol Correlator + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_SYMBOL_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run the Exact Data Correlator + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run the Exact Function Bytes + // Correlator + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run the Exact Function + // Instruction Correlator and the Exact Function Mnemonics Correlator + optionsValues.defineBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run the Duplicate Function + // Correlator + optionsValues.defineBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION_TEXT, true); + + // If true, this option will cause Auto Version Tracker to run either the Data Reference + // Correlator (if only applied Data matches exist), the Function Reference Correlator + // (if only applied Function matches exist), or the Function and Data Reference Correlator + // (if both applied data and function matches exist) + optionsValues.defineBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION_TEXT, true); + + // This option defines the minimum data length for the Exact Data Correlator + optionsValues.defineInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION_TEXT, 5); + + // This option defines the minimum symbol name lenght for the Exact Symbol Correlator + optionsValues.defineInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION_TEXT, 3); + + // This option defines the minimum function length for the Exact Function Bytes Correlator, + // the Exact Function Instruction Correlator, and the Exact Function Mnemonics Correlator + optionsValues.defineInt(VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT, 10); + + // This option defines the minimum function length for the Duplicate Function Correlator + // NOTE: This correlator can be slow on programs with large sets of duplicate function + // matches so adjusting the length can potentially speed it up + optionsValues.defineInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT, 10); + + // If true, this option will cause Implied Matches to be applied according to the limits set + // for minimum votes and maximum conflicts + optionsValues.defineBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION_TEXT, true); + + // This option defines the minimum number of votes needed to cause an implied match to be + // applied + optionsValues.defineInt(VTOptionDefines.MIN_VOTES_OPTION_TEXT, 2); + + // This option defines the maximum number of conflicts allowed to cause an implied match to + // be applied + optionsValues.defineInt(VTOptionDefines.MAX_CONFLICTS_OPTION_TEXT, 0); + + // This option defines the minimum score needed to apply reference correlator matches + optionsValues.defineDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION_TEXT, 0.95); + + // This option defines the minimum confidence level needed to apply reference correlator + // matches. Note: Due to the log10 scaling of the confidence the confidence thresholds + // equal log 10 (10*optionValue) So optionValue 1.0 = threshold 1.0, optionValue 10.0 + // = threshold 2.0, optionValue 100 = threshold 3.0 + // optionsValues.defineDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION_TEXT, 10.0); + optionsValues.defineDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION_TEXT, 10.0); + + return optionsValues; + + } +} diff --git a/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_AutoVT.html b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_AutoVT.html new file mode 100644 index 0000000000..fe4cb4b479 --- /dev/null +++ b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_AutoVT.html @@ -0,0 +1,353 @@ + + + + + + + Auto Version Tracking + + + + + +

Auto Version Tracking

+ +

The Automatic Version Tracking uses various correlators in a predetermined order + to automatically create matches, accept the most likely matches, and apply markup. The following + correlators are run in this order if the associated options are set:

+
+ + +

If the Create Implied Matches option is set, Implied Matches will be created whenever + the above correlators accept matches.

+ +

If the Apply Implied Matches option is set, any created Implied Matches that have + the set minimum number of votes and does not exceed the set maximum number of conflicts + will be applied.

+ +
+

NOTE: It is unlikely that all matches in the entire program will be made and there is no guarantee that no mistakes will be made. This + action was designed to try to save as much time as possible while also taking a conservative approach to the automation. +

+ +

+ +
+ + + +

Auto Version Tracking Options

+ +

+ Auto Version Tracking will run each selected correlator and will use the option values found + in the Auto Version Tracking options rather than the options chosen when the correlators are + run individually. To access the options use Edit->Tool Options->Version Tracking->Auto + Version Tracking from the Version Tracking Match Window.

+ +

The following options are found when the Edit->Tool Options->Version Tracking->Auto Version + Tracking options folder is selected and will determine which correlators will be run during + Auto Version Tracking.

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Create Implied MatchesWhen true, Implied Matches will be created whenever the other correlators + accept matches. + True
Run Duplicate Function CorrelatorWhen true, the Duplicate Function Instructions Correlator will be run + during Auto Version Tracking. + True
Run Exact Data CorrelatorWhen true, the Exact Data Correlator will be run during Auto Version + Tracking. + True
Run Exact Function Bytes CorrelatorWhen true, the Exact Function Bytes Correlator will be run during Auto + Version Tracking. + True
Run Exact Function Instruction CorrelatorsWhen true, the Exact Function Instruction Correlator and the Exact + Function Mnemonic Correlator will be run during Auto Version Tracking. + True
Run Exact Symbol CorrelatorWhen true, the Exact Symbol Correlator will be run during Auto Version + Tracking. + True
Run Reference CorrelatorsWhen true, the Data Reference Correlator (if previous correlators applied + only data matches), Function Reference Correlator (if previous correlators + applied only function matches), or the Combined Function and Data Reference + Correlator (if previous correlators applied both data and function matches) + will be run during Auto Version Tracking. + True
+ +
+ +

The following option is found when selecting + Edit->Tool Options->Auto Version Tracking->Data Correlator Options +

+
+
+ + + + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Data Correlator Minimum LengthThis option sets the minimum data length used to find data matches when + running the Exact Data Correlator in Auto Version Tracking. + 5
+
+ +

The following option is found when selecting + Edit->Tool Options->Auto Version Tracking->Exact Function Correlators Options +

+
+
+ + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Exact Function Correlators Minimum Function LengthThis option sets the minimum function length used to find function + matches when running the Exact Function Instruction Bytes Correlator, the + Exact Function Instruction Correlator, and the Exact Function Mnemonics + Correlator in Auto Version Tracking. + 10
+
+ +

The following option is found when selecting + Edit->Tool Options->Auto Version Tracking->Duplicate Function Correlator Options +

+
+
+ + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Duplicate Function Correlators Minimum Function LengthThis option sets the minimum function length used to find function + matches when running the Duplicate Function Instruction Correlator in + Auto Version Tracking. + 10
+
+ +

The following option is found when selecting + Edit->Tool Options->Auto Version Tracking->Symbol Correlators Options +

+
+
+ + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Symbol Correlator Minimum Symbol LengthThis option sets the the minimum symbol name length used to find function + and data matches when running the Exact Symbol Correlator in Auto Version + Tracking. + 10
+
+ + +

The following options are found when selecting + Edit->Tool Options->Auto Version Tracking->Reference Correlators Options

+
+
+ + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Reference Correlators Minimum ConfidenceThis option sets minimum confidence score used to find function matches + using the Data Reference Correlator, the Function Reference Correlator and the + Combined Function and Data Reference Correlator. + 10.0
Reference Correlators Minimum ScoreThis option sets minimum similarity score used to find function matches + using the Data Reference Correlator, the Function Reference Correlator and the + Combined Function and Data Reference Correlator. + 9.5
+
+ +

The following options are found when selecting + Edit->Tool Options->Auto Version Tracking->Implied Match Correlator Options +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescriptionDefault Value
Apply Implied MatchesThis option if true, causes Auto Version Tracking to apply Implied Matches + if the minimum vote count is met and the maximum conflict count is not exceeded. + True
Maximum Conflicts AllowedThis option sets the maximum number of allowed conflicts for an applied + implied match in Auto Version Tracking. + 0
Minimum Votes NeededThis option sets the minimum number of needed votes for an applied + implied match in Auto Version Tracking. + 2
+ + + +
+ +

Provided by: Version Tracking Plugin

+ +

Related Topics:

+ +
+
+ + diff --git a/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_Tool.html b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_Tool.html index a1b33a9003..9963191158 100644 --- a/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_Tool.html +++ b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/VT_Tool.html @@ -113,43 +113,26 @@ action will launch the Version Tracking Wizard to guide you through the process of create a new - version tracking session. + version + tracking session.

The Add to Session action will launch the Version Tracking Wizard to guide you through the process of create adding new - Program Correlator results to + Program Correlator + results to an existing - version tracking session. + version + tracking session.

-

The Automatic Version Tracking - action uses various correlators in a predetermined order - to automatically create matches, accept the most likely matches, and apply markup all with one button press. The following - correlators are run in this order: -

-
-

NOTE: It is unlikely that all matches in the entire program will be made and there is no guarantee that no mistakes will be made. This - action was designed to try to save as much time as possible while also taking a conservative approach to the automation. -

- +

The Automatic Version + Tracking action will launch + Auto Version Tracking to + try and automatically create and accept the most likely matches .

- -

The Open Session... - action will launch a session chooser dialog that allows you to pick a previously - created session. This action is available only from the File menu. -

- @@ -174,9 +157,9 @@ Version Tracking Wizard so that you can create a new version tracking session. -
  • Auto Version Track - Runs various correlators in a predetermined order - and automatically creates matches, accepts the most likely matches, and applies markup - all with one button press.
    • +
  • Auto Version Track - Runs + Auto Version Tracking to + try and automatically create and accept the most likely matches .
  • Open Session - Shows a chooser dialog that allows you to open an existing version tracking session.
    • diff --git a/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/providers/VT_Apply_Options.html b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/providers/VT_Apply_Options.html index c200601658..ed56c5791b 100644 --- a/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/providers/VT_Apply_Options.html +++ b/Ghidra/Features/VersionTracking/src/main/help/help/topics/VersionTrackingPlugin/providers/VT_Apply_Options.html @@ -606,8 +606,8 @@ - +

    Provided by: Version Tracking Plugin

    Related Topics:

    diff --git a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingAction.java b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingAction.java index 4749987e38..7e7bd7463f 100644 --- a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingAction.java +++ b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingAction.java @@ -18,7 +18,9 @@ package ghidra.feature.vt.gui.actions; import javax.swing.Icon; import docking.ActionContext; -import docking.action.*; +import docking.action.DockingAction; +import docking.action.MenuData; +import docking.action.ToolBarData; import docking.tool.ToolConstants; import generic.theme.GIcon; import ghidra.feature.vt.api.main.VTSession; @@ -27,7 +29,9 @@ import ghidra.feature.vt.gui.plugin.VTPlugin; import ghidra.framework.options.ToolOptions; import ghidra.util.HTMLUtilities; import ghidra.util.HelpLocation; -import ghidra.util.task.*; +import ghidra.util.task.Task; +import ghidra.util.task.TaskLauncher; +import ghidra.util.task.TaskListener; /** * This action runs the {@link AutoVersionTrackingTask} @@ -62,15 +66,8 @@ public class AutoVersionTrackingAction extends DockingAction { VTSession session = controller.getSession(); ToolOptions options = controller.getOptions(); - // In the future we might want to make these user options so the user can change them. - // We don't want to make this change until the confidence option in the reference - // correlators is changed to make more sense to the user - currently the confidence has - // to be entered as the value before the log 10 is computed but the table shows log 10 value. - // - // The current passed values for score and confidence (.95 and 10.0) - // get you accepted matches with similarity scores >= .95 and - // confidence (log 10) scores 2.0 and up - AutoVersionTrackingTask task = new AutoVersionTrackingTask(session, options, 0.95, 10.0); + + AutoVersionTrackingTask task = new AutoVersionTrackingTask(session, options); task.addTaskListener(new TaskListener() { @Override @@ -92,4 +89,6 @@ public class AutoVersionTrackingAction extends DockingAction { TaskLauncher.launch(task); } + + } diff --git a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java index 732b42e456..b97942e1a3 100644 --- a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java +++ b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/actions/AutoVersionTrackingTask.java @@ -71,8 +71,9 @@ import ghidra.util.task.TaskMonitor; import ghidra.util.task.WrappingTaskMonitor; /** - * This command runs all of the exact {@link VTProgramCorrelator}s that return - * unique matches (i.e., only one of each match is found in each program): + * If their options are set, this command runs all of the + * exact {@link VTProgramCorrelator}s that return unique matches (i.e., only one of each + * match is found in each program) for each correlator selected in the autoVT options to run: *
      *
    1. Exact Symbol Name correlator
      2. *
    2. Exact Data correlator
      3. @@ -81,19 +82,28 @@ import ghidra.util.task.WrappingTaskMonitor; *
    3. Exact Function Mnemonic correlator
      4. *
    * - *

    After running each correlator all matches are accepted since they are exact/unique matches - * and all markup from the source program functions is applied to the matching destination program - * functions. + *

    After running each of the above correlators all matches are accepted since they are + * exact/unique matches and all markup from the source program functions is applied to the matching + * destination program functions. * - *

    Next, this command runs the Duplicate Function Instruction correlator to find any non-unique - * functions with exact instruction bytes then compares their operands to determine and accept - * correct matches with markup. + *

    Next, if the autoVT option for this correlator is selected, the command runs the + * Duplicate Function Instruction correlator to find any non-unique + * functions with exact instruction bytes. It then compares their operands to try and determine + * unique matches within matching sets and if found will accept correct matches and apply markup. * - *

    The command then gets a little more speculative by running the Combined Function and Data - * Reference correlator, which uses match information from the previous correlators to find more - * matches. + *

    If chosen, the command then gets a little more speculative by running the Data Reference + * Correlator, the Function Reference Correlator, and/or the Combined Function and Data Reference + * Correlator, which use accepted match information from the previous correlators to find more + * matches. Only the matches with minimum score/confidence values, as chosen in the autoVT options, + * will be accepted. * - *

    As more techniques get developed, more automation will be added to this command. + *

    If the user chooses to create implied matches then whenever matches are accepted, matches + * that can be implied by those matches as new matches will be created. If the user chooses to + * accept applied matches, then they will be applied if the chosen minimum vote count is met and if + * the chosen maximum conflict count is not exceeded. + * + * All options can be set in the Version Tracking Match Window's Edit -> Tool Options in + * Version Tracking/Auto Version Tracking option folder. * */ public class AutoVersionTrackingTask extends Task { @@ -106,36 +116,30 @@ public class AutoVersionTrackingTask extends Task { private Program destinationProgram; private AddressSetView sourceAddressSet; private AddressSetView destinationAddressSet; - private double minCombinedReferenceCorrelatorScore; - private double minCombinedReferenceCorrelatorConfidence; - private ToolOptions applyOptions; + private ToolOptions toolOptions; private String statusMsg = null; private static int NUM_CORRELATORS = 8; /** * Constructor for a modal/blocking AutoVersionTrackingTask * - + * @param session The Version Tracking session containing the source, destination, correlator * and match information needed for this command. - * @param options the options used when applying matches + * @param toolOptions the options used when applying matches * @param minCombinedReferenceCorrelatorScore The minimum score used to limit matches created by * the Combined Reference Correlator. * @param minCombinedReferenceCorrelatorConfidence The minimum confidence used to limit matches * created by the Combined Reference Correlator. */ - public AutoVersionTrackingTask(VTSession session, ToolOptions options, - double minCombinedReferenceCorrelatorScore, - double minCombinedReferenceCorrelatorConfidence) { + public AutoVersionTrackingTask(VTSession session, ToolOptions toolOptions) { super(NAME, true, true, true); this.session = session; this.matchInfoFactory = new MatchInfoFactory(); this.addressCorrelator = new AddressCorrelatorManager(() -> session); this.sourceProgram = session.getSourceProgram(); this.destinationProgram = session.getDestinationProgram(); - this.minCombinedReferenceCorrelatorScore = minCombinedReferenceCorrelatorScore; - this.minCombinedReferenceCorrelatorConfidence = minCombinedReferenceCorrelatorConfidence; - this.applyOptions = options; + this.toolOptions = toolOptions; } @Override @@ -173,11 +177,12 @@ public class AutoVersionTrackingTask extends Task { int count = 0; monitor.doInitialize(NUM_CORRELATORS); - // save user option and use to determine whether to handle implied matches at all later - boolean autoCreateImpliedMatches = - applyOptions.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + // save user's Version Tracking (not AutoVT) Create implied match option - // Turn off auto implied matches and handle later if user had that option set + boolean originalImpliedMatchOption = + toolOptions.getBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + + // Turn off auto implied matches and handle later if user had that Auto VT option set // This is because when run from the VT GUI action implied matches are created automatically // by the VT controller when the option is set but they are not created when called from a // script since there is no VT controller in that case. If allowed to happen in @@ -185,50 +190,85 @@ public class AutoVersionTrackingTask extends Task { // votes will be wrong. This Task doesn't know if called from GUI or script so this is // klunky but will make sure they are only processed once and will make sure the user option // is put back the way the user had it. - applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + toolOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); - // Use default options for all of the "exact" correlators; passed in options for the others - VTOptions options; + // Start with each correlator's default options and overwrite with appropriate + // corresponding AutoVT options + VTOptions vtOptions; + String prefix = "%s correlation (%d of " + NUM_CORRELATORS + ") - "; // Run the correlators in the following order: // Do this one first because we don't want it to find ones that get markup applied by later // correlators - VTProgramCorrelatorFactory factory = new SymbolNameProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + boolean runExactSymbolCorrelator = + toolOptions.getBoolean(VTOptionDefines.RUN_EXACT_SYMBOL_OPTION, true); + if (runExactSymbolCorrelator) { + VTProgramCorrelatorFactory factory = new SymbolNameProgramCorrelatorFactory(); - String prefix = "%s correlation (%d of " + NUM_CORRELATORS + ") - "; - monitor.setPrefix(String.format(prefix, "Symbol Name", ++count)); - hasApplyErrors = correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); + vtOptions = factory.createDefaultOptions(); + int symbolMin = toolOptions.getInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION, 3); + vtOptions.setInt(SymbolNameProgramCorrelatorFactory.MIN_SYMBOL_NAME_LENGTH, symbolMin); - factory = new ExactDataMatchProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + monitor.setPrefix(String.format(prefix, "Symbol Name", ++count)); + hasApplyErrors = correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + } + boolean runExactDataCorrelator = + toolOptions.getBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION, true); + if (runExactDataCorrelator) { + VTProgramCorrelatorFactory factory = new ExactDataMatchProgramCorrelatorFactory(); - monitor.setPrefix(String.format(prefix, "Exact Data", ++count)); - hasApplyErrors |= correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); + vtOptions = factory.createDefaultOptions(); + int dataMin = toolOptions.getInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION, 5); + vtOptions.setInt(ExactDataMatchProgramCorrelatorFactory.DATA_MINIMUM_SIZE, dataMin); - factory = new ExactMatchBytesProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + monitor.setPrefix(String.format(prefix, "Exact Data", ++count)); + hasApplyErrors |= correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + } - monitor.setPrefix(String.format(prefix, "Exact Bytes", ++count)); - hasApplyErrors |= correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); + int minFunctionLen = + toolOptions.getInt(VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION, 10); - factory = new ExactMatchInstructionsProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + boolean runExactFunctionBytesCorrelator = + toolOptions.getBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION, true); + if (runExactFunctionBytesCorrelator) { + VTProgramCorrelatorFactory factory = new ExactMatchBytesProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); - monitor.setPrefix(String.format(prefix, "Exact Instructions", ++count)); - hasApplyErrors |= correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); + vtOptions.setInt(ExactMatchBytesProgramCorrelatorFactory.FUNCTION_MINIMUM_SIZE, + minFunctionLen); - factory = new ExactMatchMnemonicsProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + monitor.setPrefix(String.format(prefix, "Exact Bytes", ++count)); + hasApplyErrors |= correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + } + + boolean runExactFunctionInstCorrelator = + toolOptions.getBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION, true); + if (runExactFunctionInstCorrelator) { + VTProgramCorrelatorFactory factory = + new ExactMatchInstructionsProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + + vtOptions.setInt(ExactMatchInstructionsProgramCorrelatorFactory.FUNCTION_MINIMUM_SIZE, + minFunctionLen); + + monitor.setPrefix(String.format(prefix, "Exact Instructions", ++count)); + hasApplyErrors |= correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + + factory = new ExactMatchMnemonicsProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + + vtOptions.setInt(ExactMatchMnemonicsProgramCorrelatorFactory.FUNCTION_MINIMUM_SIZE, + minFunctionLen); + monitor.setPrefix(String.format(prefix, "Exact Mnemonic", ++count)); + hasApplyErrors |= correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + } - monitor.setPrefix(String.format(prefix, "Exact Mnemonic", ++count)); - hasApplyErrors |= correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); // This is the first of the "speculative" post-correlator match algorithm. The correlator // returns all duplicate function instruction matches so there will always be more @@ -237,12 +277,30 @@ public class AutoVersionTrackingTask extends Task { // Given that each function must contains the same instructions to even become a match, // and the compare function mechanism has been very well tested, the mechanism for // finding the correct match is very accurate. - factory = new DuplicateFunctionMatchProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); + boolean runDupeFunctionCorrelator = + toolOptions.getBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION, true); - monitor.setPrefix(String.format(prefix, "Duplicate Function", ++count)); - hasApplyErrors |= correlateAndPossiblyApplyDuplicateFunctions(factory, options, monitor); - monitor.doIncrementProgress(); + if (runDupeFunctionCorrelator) { + + VTProgramCorrelatorFactory factory = + new DuplicateFunctionMatchProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + + // if Auto VT min function length for dupe matches is different than current + // exact instruction match setting temporarily change it for auto VT run + int dupFunctionMinLen = + toolOptions.getInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION, 10); + + vtOptions.setInt( + ExactMatchInstructionsProgramCorrelatorFactory.FUNCTION_MINIMUM_SIZE, + dupFunctionMinLen); + + monitor.setPrefix(String.format(prefix, "Duplicate Function", ++count)); + hasApplyErrors |= + correlateAndPossiblyApplyDuplicateFunctions(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + + } // The rest are mores speculative matching algorithms because they depend on our // choosing the correct score/confidence pair to determine very probable matches. These @@ -251,65 +309,93 @@ public class AutoVersionTrackingTask extends Task { // Get the names of the confidence and similarity score thresholds that // are used by all of the "reference" correlators - String confidenceOption = VTAbstractReferenceProgramCorrelatorFactory.CONFIDENCE_THRESHOLD; - String scoreOption = VTAbstractReferenceProgramCorrelatorFactory.SIMILARITY_THRESHOLD; + boolean runRefCorrelators = + toolOptions.getBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION, true); + if (runRefCorrelators) { - // Get the number of data and function matches - int numDataMatches = getNumberOfDataMatches(monitor); - int numFunctionMatches = getNumberOfFunctionMatches(monitor); + double minScore = toolOptions.getDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.95); + double minConf = toolOptions.getDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); - // Run the DataReferenceCorrelator if there are accepted data matches but no accepted - // function matches - if (numDataMatches > 0 && numFunctionMatches == 0) { - factory = new DataReferenceProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); - options.setDouble(confidenceOption, minCombinedReferenceCorrelatorConfidence); - options.setDouble(scoreOption, minCombinedReferenceCorrelatorScore); - monitor.setPrefix(String.format(prefix, "Data Reference", ++count)); - hasApplyErrors = hasApplyErrors | correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); + // Get the number of data and function matches + int numDataMatches = getNumberOfDataMatches(monitor); + int numFunctionMatches = getNumberOfFunctionMatches(monitor); - // Get the number of data and function matches again if this correlator ran - numDataMatches = getNumberOfDataMatches(monitor); - numFunctionMatches = getNumberOfFunctionMatches(monitor); + // Run the DataReferenceCorrelator if there are accepted data matches but no accepted + // function matches + if (numDataMatches > 0 && numFunctionMatches == 0) { + VTProgramCorrelatorFactory factory = new DataReferenceProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.CONFIDENCE_THRESHOLD, minConf); + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.SIMILARITY_THRESHOLD, minScore); + + monitor.setPrefix(String.format(prefix, "Data Reference", ++count)); + hasApplyErrors = + hasApplyErrors | correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + + // Get the number of data and function matches again if this correlator ran + numDataMatches = getNumberOfDataMatches(monitor); + numFunctionMatches = getNumberOfFunctionMatches(monitor); + } + + // Run the FunctionReferenceCorrelator if there are accepted function matches but no + // accepted data matches + if (numDataMatches > 0 && numFunctionMatches == 0) { + VTProgramCorrelatorFactory factory = + new FunctionReferenceProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.CONFIDENCE_THRESHOLD, minConf); + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.SIMILARITY_THRESHOLD, minScore); + factory = new FunctionReferenceProgramCorrelatorFactory(); + + monitor.setPrefix(String.format(prefix, "Function Reference", ++count)); + hasApplyErrors = + hasApplyErrors | correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + + // Get the number of data and function matches again if this correlator ran + numDataMatches = getNumberOfDataMatches(monitor); + numFunctionMatches = getNumberOfFunctionMatches(monitor); + } + + // Run the CombinedDataAndFunctionReferenceCorrelator if there are both accepted function + // matches but and data matches + if (numDataMatches > 0 && numFunctionMatches > 0) { + VTProgramCorrelatorFactory factory = + new CombinedFunctionAndDataReferenceProgramCorrelatorFactory(); + vtOptions = factory.createDefaultOptions(); + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.CONFIDENCE_THRESHOLD, minConf); + vtOptions.setDouble( + VTAbstractReferenceProgramCorrelatorFactory.SIMILARITY_THRESHOLD, minScore); + + monitor.setPrefix(String.format(prefix, "Function and Data", ++count)); + hasApplyErrors = + hasApplyErrors | correlateAndPossiblyApply(factory, vtOptions, monitor); + monitor.doIncrementProgress(); + } } - // Run the FunctionReferenceCorrelator if there are accepted function matches but no - // accepted data matches - if (numDataMatches > 0 && numFunctionMatches == 0) { - factory = new FunctionReferenceProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); - options.setDouble(confidenceOption, minCombinedReferenceCorrelatorConfidence); - options.setDouble(scoreOption, minCombinedReferenceCorrelatorScore); - factory = new FunctionReferenceProgramCorrelatorFactory(); + // Use the AutoVT create implied match option to decide whether to create implied matches + // when running AutoVT + boolean autoCreateImpliedMatches = + toolOptions.getBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION, false); - monitor.setPrefix(String.format(prefix, "Function Reference", ++count)); - hasApplyErrors = hasApplyErrors | correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); - - // Get the number of data and function matches again if this correlator ran - numDataMatches = getNumberOfDataMatches(monitor); - numFunctionMatches = getNumberOfFunctionMatches(monitor); - } - - // Run the CombinedDataAndFunctionReferenceCorrelator if there are both accepted function - // matches but and data matches - if (numDataMatches > 0 && numFunctionMatches > 0) { - factory = new CombinedFunctionAndDataReferenceProgramCorrelatorFactory(); - options = factory.createDefaultOptions(); - options.setDouble(confidenceOption, minCombinedReferenceCorrelatorConfidence); - options.setDouble(scoreOption, minCombinedReferenceCorrelatorScore); - - monitor.setPrefix(String.format(prefix, "Function and Data", ++count)); - hasApplyErrors = hasApplyErrors | correlateAndPossiblyApply(factory, options, monitor); - monitor.doIncrementProgress(); - } - - // if user had implied match option chosen then figure out implied matches now - // TODO: add option for applying good matches and num votes/conflicts + // if implied matches are to be created, determine whether user wants them auto-applied + // and determine the min votes and max conflicts option limits to use if (autoCreateImpliedMatches) { - hasApplyErrors = hasApplyErrors | createImpliedMatches(true, monitor); + boolean applyImpliedMatches = + toolOptions.getBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION, true); + int minVotes = toolOptions.getInt(VTOptionDefines.MIN_VOTES_OPTION, 2); + int maxConflicts = toolOptions.getInt(VTOptionDefines.MAX_CONFLICTS_OPTION, 2); + hasApplyErrors = hasApplyErrors | + createImpliedMatches(applyImpliedMatches, minVotes, maxConflicts, monitor); } String applyMarkupStatus = " with no apply markup errors."; @@ -319,12 +405,9 @@ public class AutoVersionTrackingTask extends Task { } statusMsg = NAME + " completed successfully" + applyMarkupStatus; - - - // reset auto implied match option to user choice - // TODO: make separate AutoVT implied match option - applyOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, - autoCreateImpliedMatches); + // reset the Version Tracking auto implied match option to user choice + toolOptions.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, + originalImpliedMatchOption); } @@ -333,20 +416,21 @@ public class AutoVersionTrackingTask extends Task { * @param applyGoodMatches if true, create applied matches for "good" implied matches based on * votes/conflict information. For all the applied implied matches, rerun the creation of * applied matches until no new ones found. + * @param applyGoodMatches if true, apply matches if minVotes met and maxConflicts not exceeded + * for particular match, if false, don't apply any matches + * @param minVotes minimum votes needed to apply a match + * @param maxConflicts maximum conflicts allowed to apply a match * @param monitor the task monitor * @return true if there are any apply errors, false otherwise * @throws CancelledException if cancelled */ - private boolean createImpliedMatches(boolean applyGoodMatches, TaskMonitor monitor) + private boolean createImpliedMatches(boolean applyGoodMatches, int minVotes, int maxConflicts, + TaskMonitor monitor) throws CancelledException { Set processedSrcDestPairs = new HashSet<>(); List matchSets = session.getMatchSets(); - //TODO: make these options - int minVoteCountNeeded = 2; - int maxConflictsAllowed = 0; - monitor.setMessage("Creating Implied Matches..."); monitor.initialize(matchSets.size()); @@ -371,8 +455,8 @@ public class AutoVersionTrackingTask extends Task { VTMatchSet impliedMatchSet = session.getImpliedMatchSet(); Set goodImpliedMatches = - findGoodImpliedMatches(impliedMatchSet.getMatches(), minVoteCountNeeded, - maxConflictsAllowed, monitor); + findGoodImpliedMatches(impliedMatchSet.getMatches(), minVotes, + maxConflicts, monitor); while (goodImpliedMatches.size() > 0) { @@ -387,7 +471,7 @@ public class AutoVersionTrackingTask extends Task { // possibly find more "good" implied matches from any new implied matches found impliedMatchSet = session.getImpliedMatchSet(); goodImpliedMatches = findGoodImpliedMatches(impliedMatchSet.getMatches(), - minVoteCountNeeded, maxConflictsAllowed, monitor); + minVotes, maxConflicts, monitor); } return hasApplyErrors; @@ -612,7 +696,7 @@ public class AutoVersionTrackingTask extends Task { } ApplyMarkupItemTask markupTask = - new ApplyMarkupItemTask(session, markupItems, applyOptions); + new ApplyMarkupItemTask(session, markupItems, toolOptions); markupTask.run(monitor); boolean currentMatchHasErrors = markupTask.hasErrors(); @@ -789,7 +873,6 @@ public class AutoVersionTrackingTask extends Task { // from the initial set of related associations get all the other ones that have related // associations with all the source/destinations of the newly found associations - for (VTAssociation association : relatedAssociations) { monitor.checkCancelled(); @@ -1242,6 +1325,12 @@ public class AutoVersionTrackingTask extends Task { return offsetToOperandsMap; } + /** + * Method to create offset/operand mapping for each function in match set + * if more than one identical offset/operand mapping in src or dest piles then remove + * if no operands remove + * + */ private Map createOperandsMap(Instruction inst) { Map map = new HashMap<>(); @@ -1275,14 +1364,6 @@ public class AutoVersionTrackingTask extends Task { return map; } - - /** - * Method to create offset/operand mapping for each function in match set - * if more than one identical offset/operand mapping in src or dest piles then remove - * if no operands remove - * - */ - /** * Try to accept the given match and if can accept the match try to apply its markup * @param match The match to try and accept and apply markup to @@ -1316,7 +1397,7 @@ public class AutoVersionTrackingTask extends Task { if (markupItems != null && markupItems.size() != 0) { ApplyMarkupItemTask markupTask = - new ApplyMarkupItemTask(session, markupItems, applyOptions); + new ApplyMarkupItemTask(session, markupItems, toolOptions); markupTask.run(monitor); boolean currentMatchHasErrors = markupTask.hasErrors(); if (currentMatchHasErrors) { diff --git a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/provider/matchtable/VTMatchTableProvider.java b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/provider/matchtable/VTMatchTableProvider.java index 58fc1ada4c..e5fdef6a7f 100644 --- a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/provider/matchtable/VTMatchTableProvider.java +++ b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/provider/matchtable/VTMatchTableProvider.java @@ -15,34 +15,153 @@ */ package ghidra.feature.vt.gui.provider.matchtable; -import static ghidra.feature.vt.gui.actions.TableSelectionTrackingState.*; -import static ghidra.feature.vt.gui.plugin.VTPlugin.*; -import static ghidra.feature.vt.gui.util.VTOptionDefines.*; +import static ghidra.feature.vt.gui.actions.TableSelectionTrackingState.MAINTAIN_SELECTED_ROW_INDEX; +import static ghidra.feature.vt.gui.actions.TableSelectionTrackingState.MAINTAIN_SELECTED_ROW_VALUE; +import static ghidra.feature.vt.gui.actions.TableSelectionTrackingState.NO_SELECTION_TRACKING; +import static ghidra.feature.vt.gui.plugin.VTPlugin.FILTERED_ICON; +import static ghidra.feature.vt.gui.plugin.VTPlugin.UNFILTERED_ICON; +import static ghidra.feature.vt.gui.util.VTOptionDefines.ACCEPT_MATCH_OPTIONS_NAME; +import static ghidra.feature.vt.gui.util.VTOptionDefines.APPLY_DATA_NAME_ON_ACCEPT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.APPLY_FUNCTION_NAME_ON_ACCEPT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.APPLY_MARKUP_OPTIONS_NAME; +import static ghidra.feature.vt.gui.util.VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH; +import static ghidra.feature.vt.gui.util.VTOptionDefines.AUTO_VT_OPTIONS_NAME; +import static ghidra.feature.vt.gui.util.VTOptionDefines.CALLING_CONVENTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.CALL_FIXUP; +import static ghidra.feature.vt.gui.util.VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DATA_MATCH_DATA_TYPE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_CALLING_CONVENTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_CALL_FIXUP; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_DATA_MATCH_DATA_TYPE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_EOL_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_FUNCTION_NAME; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_FUNCTION_RETURN_TYPE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_FUNCTION_SIGNATURE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_HIGHEST_NAME_PRIORITY; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_IGNORE_EXCLUDED_MARKUP_ITEMS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_IGNORE_INCOMPLETE_MARKUP_ITEMS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_INLINE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_LABELS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_NO_RETURN; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PARAMETER_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PARAMETER_DATA_TYPES; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PARAMETER_NAMES; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PARAMETER_NAMES_REPLACE_IF_SAME_PRIORITY; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PLATE_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_POST_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_PRE_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_REPEATABLE_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DEFAULT_OPTION_FOR_VAR_ARGS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DISPLAY_APPLY_MARKUP_OPTIONS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.END_OF_LINE_COMMENT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.FUNCTION_CORRELATOR_MIN_LEN_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.FUNCTION_NAME; +import static ghidra.feature.vt.gui.util.VTOptionDefines.FUNCTION_RETURN_TYPE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.FUNCTION_SIGNATURE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.HIGHEST_NAME_PRIORITY; +import static ghidra.feature.vt.gui.util.VTOptionDefines.IGNORE_EXCLUDED_MARKUP_ITEMS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.IGNORE_INCOMPLETE_MARKUP_ITEMS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.INLINE; +import static ghidra.feature.vt.gui.util.VTOptionDefines.LABELS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.MAX_CONFLICTS_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.MIN_VOTES_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.NO_RETURN; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PARAMETER_COMMENTS; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PARAMETER_DATA_TYPES; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PARAMETER_NAMES; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PARAMETER_NAMES_REPLACE_IF_SAME_PRIORITY; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PLATE_COMMENT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.POST_COMMENT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.PRE_COMMENT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.REPEATABLE_COMMENT; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_DUPE_FUNCTION_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_EXACT_DATA_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_EXACT_FUNCTION_INST_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_EXACT_SYMBOL_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.RUN_REF_CORRELATORS_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION; +import static ghidra.feature.vt.gui.util.VTOptionDefines.VAR_ARGS; -import java.awt.*; -import java.awt.event.*; -import java.util.*; +import java.awt.Adjustable; +import java.awt.BorderLayout; +import java.awt.Dimension; +import java.awt.Rectangle; +import java.awt.event.FocusAdapter; +import java.awt.event.FocusEvent; +import java.awt.event.MouseEvent; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; -import javax.swing.*; +import javax.swing.BorderFactory; +import javax.swing.JButton; +import javax.swing.JComponent; +import javax.swing.JPanel; +import javax.swing.JScrollBar; +import javax.swing.JTable; +import javax.swing.ListSelectionModel; import javax.swing.event.ListSelectionEvent; import javax.swing.event.ListSelectionListener; -import javax.swing.table.*; +import javax.swing.table.TableCellRenderer; +import javax.swing.table.TableColumn; +import javax.swing.table.TableColumnModel; -import docking.*; -import docking.widgets.table.*; +import docking.ActionContext; +import docking.DockingWindowManager; +import docking.WindowPosition; +import docking.widgets.table.AbstractSortedTableModel; +import docking.widgets.table.GTable; +import docking.widgets.table.RowObjectSelectionManager; +import docking.widgets.table.RowObjectTableModel; +import docking.widgets.table.SelectionManager; import docking.widgets.table.threaded.ThreadedTableModel; import ghidra.feature.vt.api.impl.VTChangeManager; import ghidra.feature.vt.api.impl.VersionTrackingChangeRecord; -import ghidra.feature.vt.api.main.*; -import ghidra.feature.vt.gui.actions.*; +import ghidra.feature.vt.api.main.VTMarkupItem; +import ghidra.feature.vt.api.main.VTMatch; +import ghidra.feature.vt.api.main.VTSession; +import ghidra.feature.vt.gui.actions.AcceptMatchAction; +import ghidra.feature.vt.gui.actions.ApplyBlockedMatchAction; +import ghidra.feature.vt.gui.actions.ApplyMatchAction; +import ghidra.feature.vt.gui.actions.ChooseMatchTagAction; +import ghidra.feature.vt.gui.actions.ClearMatchAction; +import ghidra.feature.vt.gui.actions.CreateSelectionAction; +import ghidra.feature.vt.gui.actions.EditAllTagsAction; +import ghidra.feature.vt.gui.actions.MatchTableSelectionAction; +import ghidra.feature.vt.gui.actions.RejectMatchAction; +import ghidra.feature.vt.gui.actions.RemoveMatchAction; +import ghidra.feature.vt.gui.actions.RemoveMatchTagAction; +import ghidra.feature.vt.gui.actions.TableSelectionTrackingState; import ghidra.feature.vt.gui.editors.MatchTagCellEditor; -import ghidra.feature.vt.gui.filters.*; +import ghidra.feature.vt.gui.filters.AncillaryFilterDialogComponentProvider; +import ghidra.feature.vt.gui.filters.Filter; import ghidra.feature.vt.gui.filters.Filter.FilterEditingStatus; -import ghidra.feature.vt.gui.plugin.*; -import ghidra.feature.vt.gui.util.*; -import ghidra.feature.vt.gui.util.AbstractVTMatchTableModel.*; -import ghidra.framework.model.*; +import ghidra.feature.vt.gui.filters.FilterDialogModel; +import ghidra.feature.vt.gui.filters.FilterStatusListener; +import ghidra.feature.vt.gui.plugin.VTController; +import ghidra.feature.vt.gui.plugin.VTControllerListener; +import ghidra.feature.vt.gui.plugin.VTPlugin; +import ghidra.feature.vt.gui.plugin.VersionTrackingPluginPackage; +import ghidra.feature.vt.gui.util.AbstractVTMatchTableModel.DestinationLabelTableColumn; +import ghidra.feature.vt.gui.util.AbstractVTMatchTableModel.SourceLabelTableColumn; +import ghidra.feature.vt.gui.util.AbstractVTMatchTableModel.StatusTableColumn; +import ghidra.feature.vt.gui.util.AbstractVTMatchTableModel.TagTableColumn; +import ghidra.feature.vt.gui.util.AllTextFilter; +import ghidra.feature.vt.gui.util.FilterIconFlashTimer; +import ghidra.feature.vt.gui.util.MatchInfo; +import ghidra.feature.vt.gui.util.MatchStatusRenderer; +import ghidra.feature.vt.gui.util.VTSymbolRenderer; +import ghidra.framework.model.DomainObject; +import ghidra.framework.model.DomainObjectChangeRecord; +import ghidra.framework.model.DomainObjectChangedEvent; import ghidra.framework.options.Options; import ghidra.framework.options.SaveState; import ghidra.framework.plugintool.ComponentProviderAdapter; @@ -708,11 +827,74 @@ public class VTMatchTableProvider extends ComponentProviderAdapter .setOptionsHelpLocation( new HelpLocation("VersionTracking", "Apply Markup Options")); + + // Auto VT options + + // put checkboxes to determine which correlators to run during auto VT + vtOptions.registerOption(CREATE_IMPLIED_MATCHES_OPTION, true, null, + "Create Implied Matches when AutoVT correlators apply function matches."); + vtOptions.registerOption(RUN_EXACT_DATA_OPTION, true, null, "Run the Exact Data Correlator"); + vtOptions.registerOption(RUN_EXACT_SYMBOL_OPTION, true, null, "Run the Exact Symbol Correlator"); + vtOptions.registerOption(RUN_EXACT_FUNCTION_BYTES_OPTION, true, null, + "Run the Exact Function Bytes Correlator"); + vtOptions.registerOption(RUN_EXACT_FUNCTION_INST_OPTION, true, null, + "Run the Exact Function Instruction Bytes and Mnemonics Correlators"); + vtOptions.registerOption(RUN_DUPE_FUNCTION_OPTION, true, null, + "Run the Duplicate Function Instruction Correlator"); + vtOptions.registerOption(RUN_REF_CORRELATORS_OPTION, true, null, "Run the Reference Correlators"); + + // create sub options for each auto VT correlator + vtOptions.registerOption(APPLY_IMPLIED_MATCHES_OPTION, true, null, + "Apply implied matches if minimum vote count is met and maximum conflict count is not exceeded."); + vtOptions.registerOption(MIN_VOTES_OPTION, 2, null, + "The minimum number of votes needed to apply an implied match."); + vtOptions.registerOption(MAX_CONFLICTS_OPTION, 0, null, + "The maximum number of conflicts allowed to apply an implied match."); + + vtOptions.registerOption(SYMBOL_CORRELATOR_MIN_LEN_OPTION, 3, null, + "Minimum Symbol Name Length of Auto Version Tracking Symbol Correlator"); + vtOptions.getOptions(SYMBOL_CORRELATOR_MIN_LEN_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + + vtOptions.registerOption(DATA_CORRELATOR_MIN_LEN_OPTION, 5, null, + "Minimum Data Length of Auto Version Tracking Data Correlator"); + vtOptions.getOptions(DATA_CORRELATOR_MIN_LEN_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + + vtOptions.registerOption(FUNCTION_CORRELATOR_MIN_LEN_OPTION, 10, null, + "Minimum Function Length of Auto Version Tracking Duplicate Function Correlator"); + vtOptions.getOptions(FUNCTION_CORRELATOR_MIN_LEN_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + + vtOptions.registerOption(DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION, 10, null, + "Minimum Function Length of Auto Version Tracking Duplicate Function Correlator"); + vtOptions.getOptions(DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + + vtOptions.registerOption(REF_CORRELATOR_MIN_SCORE_OPTION, 0.95, null, + "Minimum Score of all Auto Version Tracking Reference Function Correlators (Data, Function, and Combined Function and Data)"); + vtOptions.getOptions(REF_CORRELATOR_MIN_SCORE_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + + vtOptions.registerOption(REF_CORRELATOR_MIN_CONF_OPTION, 10.0, null, + "Minimum Confidence of all Auto Version Tracking Reference Function Correlator (Data, Function, and Combined Function and Data)"); + vtOptions.getOptions(REF_CORRELATOR_MIN_CONF_OPTION) + .setOptionsHelpLocation( + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options")); + HelpLocation applyOptionsHelpLocation = new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Version_Tracking_Apply_Options"); HelpLocation applyMatchOptionsHelpLocation = new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Match_Apply_Options"); + HelpLocation autoVTHelpLocation = + new HelpLocation(VTPlugin.HELP_TOPIC_NAME, "Auto_Version_Tracking_Options"); + vtOptions.setOptionsHelpLocation(applyOptionsHelpLocation); vtOptions.getOptions(ACCEPT_MATCH_OPTIONS_NAME) @@ -720,6 +902,9 @@ public class VTMatchTableProvider extends ComponentProviderAdapter vtOptions.getOptions(APPLY_MARKUP_OPTIONS_NAME) .setOptionsHelpLocation(applyMatchOptionsHelpLocation); + + vtOptions.setOptionsHelpLocation(autoVTHelpLocation); + vtOptions.getOptions(AUTO_VT_OPTIONS_NAME).setOptionsHelpLocation(autoVTHelpLocation); } //================================================================================================== diff --git a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/util/VTOptionDefines.java b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/util/VTOptionDefines.java index d8909088f0..1ff621ca86 100644 --- a/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/util/VTOptionDefines.java +++ b/Ghidra/Features/VersionTracking/src/main/java/ghidra/feature/vt/gui/util/VTOptionDefines.java @@ -15,7 +15,16 @@ */ package ghidra.feature.vt.gui.util; -import ghidra.feature.vt.gui.util.VTMatchApplyChoices.*; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.CallingConventionChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.CommentChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.FunctionNameChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.FunctionSignatureChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.HighestSourcePriorityChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.LabelChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.ParameterDataTypeChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.ReplaceChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.ReplaceDataChoices; +import ghidra.feature.vt.gui.util.VTMatchApplyChoices.SourcePriorityChoices; import ghidra.framework.options.Options; public class VTOptionDefines { @@ -30,7 +39,6 @@ public class VTOptionDefines { ".Automatically Apply Data Label on Accept"; // Apply Options - public static final String APPLY_MARKUP_OPTIONS_NAME = "Apply Markup Options"; public static boolean DEFAULT_OPTION_FOR_IGNORE_INCOMPLETE_MARKUP_ITEMS = false; public static boolean DEFAULT_OPTION_FOR_IGNORE_EXCLUDED_MARKUP_ITEMS = false; @@ -108,4 +116,101 @@ public class VTOptionDefines { public final static String DISPLAY_APPLY_MARKUP_OPTIONS = APPLY_MARKUP_OPTIONS_NAME + Options.DELIMITER + "Display Apply Markup Options"; + + // Auto VT Options + public static final String AUTO_VT_OPTIONS_NAME = "Auto Version Tracking Options"; + + public static final String AUTO_VT_SYMBOL_CORRELATOR = "Symbol Correlator Options"; + public static final String AUTO_VT_DATA_CORRELATOR = "Data Correlator Options"; + public static final String AUTO_VT_EXACT_FUNCTION_CORRELATORS = + "Exact Function Correlators Options"; + public static final String AUTO_VT_DUPLICATE_FUNCTION_CORRELATOR = + "Duplicate Function Correlator Options"; + public static final String AUTO_VT_REFERENCE_CORRELATORS = + "Reference Correlators Options"; + public static final String AUTO_VT_IMPLIED_MATCH_CORRELATOR = + "Implied Match Correlator Options"; + + public static final String CREATE_IMPLIED_MATCHES_OPTION_TEXT = "Create Implied Matches"; + + public static final String CREATE_IMPLIED_MATCHES_OPTION = AUTO_VT_OPTIONS_NAME + + "." + CREATE_IMPLIED_MATCHES_OPTION_TEXT; + + public static final String RUN_EXACT_DATA_OPTION_TEXT = "Run Exact Data Correlator"; + public static final String RUN_EXACT_DATA_OPTION = AUTO_VT_OPTIONS_NAME + + "." + RUN_EXACT_DATA_OPTION_TEXT; + + public static final String RUN_EXACT_SYMBOL_OPTION_TEXT = "Run Exact Symbol Correlator"; + public static final String RUN_EXACT_SYMBOL_OPTION = AUTO_VT_OPTIONS_NAME + + "." + RUN_EXACT_SYMBOL_OPTION_TEXT; + + public static final String RUN_EXACT_FUNCTION_BYTES_OPTION_TEXT = + "Run Exact Function Bytes Correlator"; + public static final String RUN_EXACT_FUNCTION_BYTES_OPTION = AUTO_VT_OPTIONS_NAME + + "." + "Run Exact Function Bytes Correlator"; + + public static final String RUN_EXACT_FUNCTION_INST_OPTION_TEXT = + "Run Exact Function Instructions Correlators"; + public static final String RUN_EXACT_FUNCTION_INST_OPTION = AUTO_VT_OPTIONS_NAME + + "." + RUN_EXACT_FUNCTION_INST_OPTION_TEXT; + + public static final String RUN_DUPE_FUNCTION_OPTION_TEXT = "Run Duplicate Function Correlator"; + public static final String RUN_DUPE_FUNCTION_OPTION = AUTO_VT_OPTIONS_NAME + + "." + RUN_DUPE_FUNCTION_OPTION_TEXT; + + public static final String RUN_REF_CORRELATORS_OPTION_TEXT = "Run the Reference Correlators"; + public static final String RUN_REF_CORRELATORS_OPTION = AUTO_VT_OPTIONS_NAME + + "." + RUN_REF_CORRELATORS_OPTION_TEXT; + + public static final String APPLY_IMPLIED_MATCHES_OPTION_TEXT = "Apply Implied Matches"; + + public static final String APPLY_IMPLIED_MATCHES_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_IMPLIED_MATCH_CORRELATOR + + "." + APPLY_IMPLIED_MATCHES_OPTION_TEXT; + + public static final String MIN_VOTES_OPTION_TEXT = "Minimum Votes Needed"; + public static final String MIN_VOTES_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_IMPLIED_MATCH_CORRELATOR + + "." + MIN_VOTES_OPTION_TEXT; + + public static final String MAX_CONFLICTS_OPTION_TEXT = "Maximum Conflicts Allowed"; + public static final String MAX_CONFLICTS_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_IMPLIED_MATCH_CORRELATOR + + "." + MAX_CONFLICTS_OPTION_TEXT; + + public static final String SYMBOL_CORRELATOR_MIN_LEN_OPTION_TEXT = + "Symbol Correlator Minimum Symbol Length"; + public static final String SYMBOL_CORRELATOR_MIN_LEN_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_SYMBOL_CORRELATOR + + "." + SYMBOL_CORRELATOR_MIN_LEN_OPTION_TEXT; + + public static final String DATA_CORRELATOR_MIN_LEN_OPTION_TEXT = + "Data Correlator Minimum Data Length"; + public static final String DATA_CORRELATOR_MIN_LEN_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_DATA_CORRELATOR + + "." + "Data Correlator Minimum Data Length"; + + public static final String FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT = + "Exact Function Correlators Minimum Function Length"; + public static final String FUNCTION_CORRELATOR_MIN_LEN_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_EXACT_FUNCTION_CORRELATORS + + "." + FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT; + + public static final String DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT = + "Duplicate Function Correlator Minimum Function Length"; + public static final String DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_DUPLICATE_FUNCTION_CORRELATOR + + "." + DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION_TEXT;; + + public static final String REF_CORRELATOR_MIN_SCORE_OPTION_TEXT = + "Reference Correlators Minimum Score"; + public static final String REF_CORRELATOR_MIN_SCORE_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_REFERENCE_CORRELATORS + + "." + REF_CORRELATOR_MIN_SCORE_OPTION_TEXT; + + public static final String REF_CORRELATOR_MIN_CONF_OPTION_TEXT = + "Reference Correlators Minimum Confidence"; + public static final String REF_CORRELATOR_MIN_CONF_OPTION = + AUTO_VT_OPTIONS_NAME + "." + AUTO_VT_REFERENCE_CORRELATORS + + "." + REF_CORRELATOR_MIN_CONF_OPTION_TEXT; } diff --git a/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java b/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java index ba5575c962..98f45ecd67 100644 --- a/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java +++ b/Ghidra/Features/VersionTracking/src/test.slow/java/ghidra/feature/vt/api/VTAutoVersionTrackingTest.java @@ -18,8 +18,8 @@ package ghidra.feature.vt.api; import static ghidra.feature.vt.db.VTTestUtils.addr; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.Collection; @@ -42,6 +42,7 @@ import ghidra.feature.vt.api.main.VTProgramCorrelator; import ghidra.feature.vt.api.main.VTScore; import ghidra.feature.vt.api.main.VTSession; import ghidra.feature.vt.api.util.VTAssociationStatusException; +import ghidra.feature.vt.api.util.VTOptions; import ghidra.feature.vt.db.VTTestUtils; import ghidra.feature.vt.gui.VTTestEnv; import ghidra.feature.vt.gui.actions.AutoVersionTrackingTask; @@ -49,18 +50,21 @@ import ghidra.feature.vt.gui.plugin.VTController; import ghidra.feature.vt.gui.util.VTOptionDefines; import ghidra.framework.options.Options; import ghidra.framework.options.ToolOptions; +import ghidra.framework.plugintool.PluginTool; import ghidra.program.database.ProgramDB; import ghidra.program.database.function.OverlappingFunctionException; import ghidra.program.model.address.Address; import ghidra.program.model.address.AddressSet; import ghidra.program.model.listing.CodeUnit; import ghidra.program.model.listing.CodeUnitIterator; +import ghidra.program.model.listing.Data; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.FunctionManager; import ghidra.program.model.listing.Listing; import ghidra.program.model.listing.Program; import ghidra.program.model.mem.MemoryAccessException; import ghidra.program.model.symbol.SourceType; +import ghidra.program.model.symbol.Symbol; import ghidra.test.AbstractGhidraHeadedIntegrationTest; import ghidra.util.Msg; import ghidra.util.exception.InvalidInputException; @@ -72,7 +76,6 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe private static final String TEST_DESTINATION_PROGRAM_NAME = "VersionTracking/WallaceVersion2"; private VTTestEnv env; - private VTController controller; private ProgramDB sourceProgram; private ProgramDB destinationProgram; private VTSessionDB session; @@ -102,11 +105,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up - runAutoVTCommand(0.999999999, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // verify that the default options are what we expect // if this assert fails then the follow-on tests will probably fail @@ -162,12 +169,16 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // Score 0.5 and conf threshold 1.0 allow similarity scores of higher than 0.5 for combined // reference correlator and 1.0 and higher for the log 10 confidence score - runAutoVTCommand(0.5, 1.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.5); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 1.0); + + runAutoVTCommand(vtOptions); // verify that the default options are what we expect // if this assert fails then the follow-on tests will probably fail @@ -213,8 +224,7 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // This test is testing to make sure that previously blocked matches do not become // accepted matches after running Auto VT @@ -235,7 +245,12 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe assertEquals(status2, VTAssociationStatus.ACCEPTED); // run auto VT which would normally accept the match we blocked - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Now test that the match we blocked is still blocked to verify that auto VT // does not accept blocked matches @@ -245,6 +260,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe VTAssociationStatus.BLOCKED); } + private ToolOptions getVTToolOptions(PluginTool tool) { + ToolOptions vtOptions = new VTOptions("Dummy"); + + if (tool != null) { + vtOptions = tool.getOptions(VTController.VERSION_TRACKING_OPTIONS_NAME); + } + return vtOptions; + } + @Test public void testDuplicateMatches_DifferentRegisterOperands_allUnique() throws Exception { @@ -270,11 +294,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // run auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Now test that the correct matches were created based on the duplicate functions we created String correlator = "Duplicate Function Instructions Match"; @@ -316,11 +344,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // run auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Now test that the correct matches were created based on the duplicate functions we created String correlator = "Duplicate Function Instructions Match"; @@ -352,11 +384,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // run auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Test to make sure that they weren't matched by something else first so we can // be sure that we are testing just the duplicate test case @@ -421,11 +457,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // run auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Test to make sure that they weren't matched by something else first so we can // be sure that we are testing just duplicate correlator match case @@ -575,11 +615,15 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // run auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Test to make sure that they weren't matched by something else first so we can // be sure that we are testing just the duplicate test case @@ -611,8 +655,7 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // Put some markup in the tested source function EOL comments Listing sourceListing = sourceProgram.getListing(); @@ -633,7 +676,12 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe sourceProgram.endTransaction(startTransaction, true); // run Auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Check that the match we are interested in got accepted String correlator = "Combined Function and Data Reference Match"; @@ -667,8 +715,7 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // Put some markup in the tested source function EOL comments Listing sourceListing = sourceProgram.getListing(); @@ -689,7 +736,12 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe sourceProgram.endTransaction(startTransaction, true); // run Auto VT - runAutoVTCommand(1.0, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 1.0); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + + runAutoVTCommand(vtOptions); // Check that the match we are interested in got accepted String correlator = "Duplicate Function Instructions Match"; @@ -748,12 +800,16 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); + PluginTool tool = env.showTool(); // Now run the AutoVT command with lower confidence thresholds to allow the match we want to // test in as a match - runAutoVTCommand(0.5, 1.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.5); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 1.0); + + runAutoVTCommand(vtOptions); // Check that the match we are interested in got accepted String correlator = "Combined Function and Data Reference Match"; @@ -812,13 +868,20 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); - ToolOptions options = controller.getOptions(); - options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, true); + PluginTool tool = env.showTool(); // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up - runAutoVTCommand(0.999999999, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION, true); + vtOptions.setBoolean(VTOptionDefines.APPLY_IMPLIED_MATCHES_OPTION, true); + vtOptions.setInt(VTOptionDefines.MIN_VOTES_OPTION, 3); + vtOptions.setInt(VTOptionDefines.MAX_CONFLICTS_OPTION, 0); + + + runAutoVTCommand(vtOptions); VTMatchSet impliedMatchSet = session.getImpliedMatchSet(); assertTrue(impliedMatchSet.getMatchCount() > 0); @@ -837,10 +900,9 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe VTAssociation association = match.getAssociation(); int numConflicts = association.getRelatedAssociations().size() - 1; - // TODO: use options once options created // if not min vote count or has conflicts - make sure not accepted match - if (association.getVoteCount() < 2 || numConflicts > 0) { + if (association.getVoteCount() < 3 || numConflicts > 0) { assertEquals(VTAssociationStatus.AVAILABLE, matchStatus); continue; } @@ -860,17 +922,208 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe session = env.createSession(sourceProgram, destinationProgram); - env.showTool(); - controller = env.getVTController(); - ToolOptions options = controller.getOptions(); - options.setBoolean(VTOptionDefines.AUTO_CREATE_IMPLIED_MATCH, false); + PluginTool tool = env.showTool(); // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up - runAutoVTCommand(0.999999999, 10.0); + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setBoolean(VTOptionDefines.CREATE_IMPLIED_MATCHES_OPTION, false); + + runAutoVTCommand(vtOptions); assertTrue(session.getImpliedMatchSet().getMatchCount() == 0); } + /* + * This tests auto version tracking with some correlators not set to run + */ + @Test + public void testRunAutoVT_disableSomeCorrelators() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + PluginTool tool = env.showTool(); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION, false); + vtOptions.setBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION, false); + + runAutoVTCommand(vtOptions); + + VTMatchSet dataMatchSet = getVTMatchSet(session, "Exact Data Match"); + assertNull(dataMatchSet); + + VTMatchSet dupMatchSet = getVTMatchSet(session, "Duplicate Function Instructions Match"); + assertNull(dupMatchSet); + + } + + /* + * This tests auto version tracking with higher min dupe function len + */ + @Test + public void testRunAutoVT_changeMinFunctionLength() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + PluginTool tool = env.showTool(); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setInt(VTOptionDefines.DUPE_FUNCTION_CORRELATOR_MIN_LEN_OPTION, 20); + + runAutoVTCommand(vtOptions); + + VTMatchSet dupMatchSet = getVTMatchSet(session, "Duplicate Function Instructions Match"); + assertNotNull(dupMatchSet); + + for (VTMatch match : dupMatchSet.getMatches()) { + + VTAssociationStatus matchStatus = + getMatchStatus(session, "Duplicate Function Instructions Match", + match.getSourceAddress(), match.getDestinationAddress()); + + if (matchStatus == VTAssociationStatus.BLOCKED) { + continue; + } + + int sourceLength = match.getSourceLength(); + int destinationLength = match.getDestinationLength(); + + if (matchStatus == VTAssociationStatus.AVAILABLE) { + assertTrue(sourceLength < 20); + assertTrue(destinationLength < 20); + continue; + } + if (matchStatus == VTAssociationStatus.ACCEPTED) { + assertTrue(sourceLength >= 20); + assertTrue(destinationLength >= 20); + } + } + + } + + /* + * This tests auto version tracking with higher min symbol len + */ + @Test + public void testRunAutoVT_changeMinSymbolLength() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + PluginTool tool = env.showTool(); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setBoolean(VTOptionDefines.RUN_EXACT_DATA_OPTION, false); + vtOptions.setBoolean(VTOptionDefines.RUN_DUPE_FUNCTION_OPTION, false); + vtOptions.setBoolean(VTOptionDefines.RUN_EXACT_FUNCTION_BYTES_OPTION, false); + vtOptions.setBoolean(VTOptionDefines.RUN_REF_CORRELATORS_OPTION, false); + + vtOptions.setInt(VTOptionDefines.SYMBOL_CORRELATOR_MIN_LEN_OPTION, 7); + + runAutoVTCommand(vtOptions); + + VTMatchSet matchSet = getVTMatchSet(session, "Exact Symbol Name Match"); + assertNotNull(matchSet); + + for (VTMatch match : matchSet.getMatches()) { + + VTAssociationStatus matchStatus = getMatchStatus(session, "Exact Symbol Name Match", + match.getSourceAddress(), match.getDestinationAddress()); + + if (matchStatus == VTAssociationStatus.BLOCKED) { + continue; + } + + Address sourceAddress = match.getSourceAddress(); + Symbol primarySymbol = sourceProgram.getSymbolTable().getPrimarySymbol(sourceAddress); + + int length = primarySymbol.getName().length(); + if (matchStatus == VTAssociationStatus.AVAILABLE) { + assertTrue(length < 7); + continue; + } + if (matchStatus == VTAssociationStatus.ACCEPTED) { + assertTrue(length >= 7); + } + } + + } + + /* + * This tests auto version tracking with higher min data len + */ + @Test + public void testRunAutoVT_changeMinDataLength() throws Exception { + + sourceProgram = env.getProgram(TEST_SOURCE_PROGRAM_NAME); + destinationProgram = env.getProgram(TEST_DESTINATION_PROGRAM_NAME); + + session = env.createSession(sourceProgram, destinationProgram); + + PluginTool tool = env.showTool(); + + // Score .999999 and confidence 10.0 (log10 confidence 2.0) and up + ToolOptions vtOptions = getVTToolOptions(tool); + + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_SCORE_OPTION, 0.999999999); + vtOptions.setDouble(VTOptionDefines.REF_CORRELATOR_MIN_CONF_OPTION, 10.0); + vtOptions.setInt(VTOptionDefines.DATA_CORRELATOR_MIN_LEN_OPTION, 10); + + runAutoVTCommand(vtOptions); + + VTMatchSet matchSet = getVTMatchSet(session, "Exact Data Match"); + assertNotNull(matchSet); + + for (VTMatch match : matchSet.getMatches()) { + + VTAssociationStatus matchStatus = getMatchStatus(session, "Exact Data Match", + match.getSourceAddress(), match.getDestinationAddress()); + + if (matchStatus == VTAssociationStatus.BLOCKED) { + continue; + } + + Address sourceAddress = match.getSourceAddress(); + Data data = sourceProgram.getListing().getDataAt(sourceAddress); + + assertNotNull(data); + + int length = data.getLength(); + + if (matchStatus == VTAssociationStatus.AVAILABLE) { + assertTrue(length < 10); + continue; + } + if (matchStatus == VTAssociationStatus.ACCEPTED) { + assertTrue(length >= 10); + } + } + + } + private VTMatch createMatch(Address sourceAddress, Address destinationAddress, boolean setAccepted) throws VTAssociationStatusException { VTProgramCorrelator correlator = @@ -916,11 +1169,10 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe assertEquals(expectedAcceptedMatchCount, getNumAcceptedMatches(vtSession, correlatorName)); } - private void runAutoVTCommand(double minReferenceCorrelatorScore, - double minReferenceCorrelatorConfidence) { + private void runAutoVTCommand(ToolOptions options) { - AutoVersionTrackingTask task = new AutoVersionTrackingTask(session, controller.getOptions(), - minReferenceCorrelatorScore, minReferenceCorrelatorConfidence); + AutoVersionTrackingTask task = + new AutoVersionTrackingTask(session, options); TaskLauncher.launch(task); waitForSession(); } @@ -940,8 +1192,7 @@ public class VTAutoVersionTrackingTest extends AbstractGhidraHeadedIntegrationTe } } - fail("Unable to find a match set for '" + correlatorName + "'"); - return null; /// can't get here + return null; } private boolean assertCorrectScoreAndConfidenceValues(VTSession vtSession,