diff --git a/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java b/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java index 35aa6bf3e5..bde8d5e6b6 100644 --- a/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java +++ b/Ghidra/Features/FunctionID/ghidra_scripts/FidStatistics.java @@ -27,6 +27,7 @@ import ghidra.program.database.ProgramContentHandler; import ghidra.program.model.lang.Language; import ghidra.program.model.listing.*; import ghidra.program.model.mem.MemoryAccessException; +import ghidra.program.model.symbol.Symbol; import ghidra.util.exception.CancelledException; import ghidra.util.exception.VersionException; @@ -64,7 +65,9 @@ public class FidStatistics extends GhidraScript { @Override public int compareTo(SymbolPair o) { int val = sym1.compareTo(o.sym1); - if (val != 0) return val; + if (val != 0) { + return val; + } return sym2.compareTo(o.sym2); } @@ -118,6 +121,7 @@ public class FidStatistics extends GhidraScript { public int totalFunction; public int matchUniquely; public int matchMultiply; + public int hitCount; public int noMatch; public int nameMatched; public int falsePositive; @@ -126,14 +130,16 @@ public class FidStatistics extends GhidraScript { totalFunction = 0; matchUniquely = 0; matchMultiply = 0; + hitCount = 0; noMatch = 0; nameMatched = 0; falsePositive = 0; } public static void indent(StringBuilder buf,String last) { - for(int i=last.length();i<10;++i) + for(int i=last.length();i<10;++i) { buf.append(' '); + } } public void print(StringBuilder buf) { @@ -143,9 +149,12 @@ public class FidStatistics extends GhidraScript { str = Integer.toString(noMatch); buf.append(str); indent(buf,str); - str = Integer.toString(matchUniquely+matchMultiply); + str = Integer.toString(matchUniquely + matchMultiply); buf.append(str); - indent(buf,str); + indent(buf, str); + str = Integer.toString(hitCount); + buf.append(str); + indent(buf, str); str = '(' + Integer.toString(matchUniquely) + ',' + Integer.toString(matchMultiply) + ')'; buf.append(str); indent(buf,str); @@ -157,7 +166,7 @@ public class FidStatistics extends GhidraScript { } public static String getColumns() { - return "Total No Match Hits uniq/mult N-Match False"; + return "Total No Match Possible Hits uniq/mult N-Match False"; } } @@ -331,6 +340,8 @@ public class FidStatistics extends GhidraScript { addEquivSymbols("WPP_SF_qqDD","WPP_SF_qqdd"); addEquivSymbols("WPP_SF_DqD","WPP_SF_dqd"); addEquivSymbols("WPP_SF_Dq","WPP_SF_dq"); + addEquivSymbols("std::vector<>::_Assign_rv", "std::vector<>::_Move_assign_from"); + addEquivSymbols("std::vector<>::_Assign_rv", "std::vector<>::_Move_from"); } private void findDomainFiles(LinkedList programs, DomainFolder folder) @@ -350,11 +361,25 @@ public class FidStatistics extends GhidraScript { } private boolean checkNames(String a,String b) { - if (a.equals(b)) + if (a.equals(b)) { return true; + } return equivSymbols.contains(new SymbolPair(a,b)); } + private String chooseFunctionName(FidSearchResult result) { + Program program = result.function.getProgram(); + Symbol[] symbols = program.getSymbolTable().getSymbols(result.function.getEntryPoint()); + if (symbols.length > 1) { + for (Symbol symbol : symbols) { + if (matchAnalysis.containsRawName(symbol.getName())) { + return symbol.getName(); + } + } + } + return result.function.getName(); + } + private void processFunctionResult(FidSearchResult result) throws CancelledException, IOException { StatRecord record = statRecord; @@ -378,7 +403,8 @@ public class FidStatistics extends GhidraScript { matchHappened = true; // FID will put down a single match finalMatchName = matchAnalysis.getNameIterator().next(); } - NameVersions nameVersions = NameVersions.generate(result.function.getName(), program); + String funcName = chooseFunctionName(result); + NameVersions nameVersions = NameVersions.generate(funcName, program); String strippedTemplateName = null; if (nameVersions.demangledBaseName != null) { strippedTemplateName = @@ -389,7 +415,9 @@ public class FidStatistics extends GhidraScript { while(iter.hasNext()) { String raw = iter.next(); NameVersions matchNames = NameVersions.generate(raw, program); - if (matchNames.rawName == null) continue; + if (matchNames.rawName == null) { + continue; + } if (checkNames(nameVersions.rawName,matchNames.rawName)) { exactNameMatch = true; break; @@ -398,8 +426,26 @@ public class FidStatistics extends GhidraScript { exactNameMatch = true; break; } - if (nameVersions.demangledBaseName == null) continue; - if (matchNames.demangledBaseName == null) continue; + if (nameVersions.demangledBaseName != null) { + if (checkNames(nameVersions.demangledBaseName, matchNames.rawName) || + checkNames(nameVersions.demangledBaseName, matchNames.similarName)) { + exactNameMatch = true; + break; + } + } + if (matchNames.demangledBaseName != null) { + if (checkNames(nameVersions.rawName, matchNames.demangledBaseName) || + checkNames(nameVersions.similarName, matchNames.demangledBaseName)) { + exactNameMatch = true; + break; + } + } + if (nameVersions.demangledBaseName == null) { + continue; + } + if (matchNames.demangledBaseName == null) { + continue; + } if (checkNames(nameVersions.demangledBaseName,matchNames.demangledBaseName)) { exactNameMatch = true; break; @@ -408,17 +454,21 @@ public class FidStatistics extends GhidraScript { String strippedName = MatchNameAnalysis.removeTemplateParams(matchNames.demangledBaseName); if (strippedName != null) { - if (strippedName.equals(strippedTemplateName)) { + if (checkNames(strippedName, strippedTemplateName)) { exactNameMatch = true; break; } } } } - if (exactNameMatch) + if (exactNameMatch) { record.nameMatched += 1; + } float score = result.matches.get(0).getOverallScore(); + if (score >= scoreThreshold && matchHappened) { + record.hitCount += 1; + } if (exactNameMatch && ((score < scoreThreshold) || !matchHappened)) { MatchRecord matchRecord = new MatchRecord(result,null,false); StringBuilder buffer = new StringBuilder(); @@ -440,13 +490,19 @@ public class FidStatistics extends GhidraScript { private void processProgram(Program program,FidQueryService queryService) throws MemoryAccessException, CancelledException, VersionException, IOException { FidProgramSeeker programSeeker = service.getProgramSeeker(program,queryService, 10.0f); + monitor.setMessage("Processing " + program.getName()); + monitor.initialize(program.getFunctionManager().getFunctionCount()); FunctionIterator iter = program.getFunctionManager().getFunctionsNoStubs(true); while(iter.hasNext()) { Function func = iter.next(); - if (func.getName().startsWith("FUN_") || func.getName().startsWith("Ordinal_")) + monitor.incrementProgress(1); + if (func.getName().startsWith("FUN_") || func.getName().startsWith("Ordinal_")) { continue; + } FidSearchResult searchResult = programSeeker.searchFunction(func, monitor); - if (searchResult == null) continue; // Could not hash function + if (searchResult == null) { + continue; // Could not hash function + } processFunctionResult(searchResult); } } @@ -496,8 +552,9 @@ public class FidStatistics extends GhidraScript { try { program = (Program) domainFile.getDomainObject(this, false, false, monitor); if (queryService == null || !lastLanguage.equals(program.getLanguage())) { - if (queryService != null) + if (queryService != null) { queryService.close(); + } lastLanguage = program.getLanguage(); queryService = service.openFidQueryService(lastLanguage, false); } @@ -508,8 +565,9 @@ public class FidStatistics extends GhidraScript { monitor.setProgress(counter); } finally { - if (program != null) + if (program != null) { program.release(this); + } } } } catch(CancelledException ex) { diff --git a/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java b/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java index eb10567003..e2d2bd9e83 100644 --- a/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java +++ b/Ghidra/Features/FunctionID/ghidra_scripts/RemoveFunctions.java @@ -28,6 +28,7 @@ import ghidra.feature.fid.db.*; */ public class RemoveFunctions extends GhidraScript { private LinkedList> REMOVE_HASHES = new LinkedList<>(); + private LinkedList> REMOVE_SPECHASHES = new LinkedList<>(); private LinkedList> FORCE_SPECIFIC = new LinkedList<>(); private LinkedList> FORCE_RELATION = new LinkedList<>(); private LinkedList> AUTO_PASS = new LinkedList<>(); @@ -146,6 +147,7 @@ public class RemoveFunctions extends GhidraScript { FORCE_RELATION.add(fh(15, 0x3cd4904368bf315cL)); // ??_G?$_MallocaArrayHolder@PAVContext@Concurrency@@@details@Concurrency@@UAEPAXI@Z FORCE_RELATION.add(fh(4, 0xb8db1dacc3441a8fL)); // ??1_Timer@details@Concurrency@@MAE@XZ FORCE_RELATION.add(fh(14, 0x7b2255d33cddad65L)); // ??_GThreadInternalContext@details@Concurrency@@UAEPAXI@Z + FORCE_RELATION.add(fh(15, 0xfe727990231ca423L)); // Generic constructor REMOVE_HASHES.add(fh(4, 0x8f0554c0936e0e0dL)); // ?AddPaneToList@CPaneContainerManager@@QAEXPAVCDockablePane@@@Z REMOVE_HASHES.add(fh(17, 0x6875ba2bfa94ae88L)); // ??1?$CComPtrBase@UIAccessibleProxy@@@ATL@@QAE@XZ @@ -261,6 +263,47 @@ public class RemoveFunctions extends GhidraScript { REMOVE_HASHES.add(fh(23, 0x1c009fbde7812ed5L)); // ??0failure@ios_base@std@@QEAA@AEBV012@@Z REMOVE_HASHES.add(fh(10, 0x7e10fbe69b976818L)); // ??1CAudioMediaType@@MEAA@XZ REMOVE_HASHES.add(fh(15, 0x79c797eb8032b47L)); // ??_G?$CList@IAEAI@@UEAAPEAXI@Z + REMOVE_HASHES.add(fh(6, 0x1198931964e874fbL)); // ??0XRibbonInfoParserRoot@CMFCRibbonInfo@@IEAA@XZ + REMOVE_HASHES.add(fh(6, 0x1524536d78c0da92L)); // ??1?$shared_ptr@V__ExceptionPtr@@@std@@QEAA@XZ + REMOVE_HASHES.add(fh(7, 0x020d6fdf4571e246L)); // max_size + REMOVE_HASHES.add(fh(13, 0x225ba93763be05b1L)); // operator== + REMOVE_HASHES.add(fh(13, 0x287ae8c33777713cL)); // ??1CStreamOnCString@@UEAA@XZ + REMOVE_HASHES.add(fh(9, 0x288020713ca1ea5bL)); // ??0?$move_iterator@V?$_Vector_iterator@V?$_Vector_val@U?$_Simple_types@V?$shared_ptr@U?$_Task_impl@E@details@Concurrency@@@std@@@std@@@std@@@std@@@std@@QEAA@AEBV01@@Z + REMOVE_HASHES.add(fh(10, 0x2b3bd58383bd38e9L)); // ??1VirtualProcessorRoot@details@Concurrency@@UEAA@XZ + REMOVE_HASHES.add(fh(7, 0x30a80a215b0bb2c4L)); // Generic destructor + REMOVE_HASHES.add(fh(7, 0x53a0821862961cb4L)); // constructor + REMOVE_HASHES.add(fh(4, 0x556c824f94bb014fL)); // return value derefed from first parameter ptr + REMOVE_HASHES.add(fh(12, 0x6581a15f3efa819eL)); // copy two fields between pointer parameters + REMOVE_HASHES.add(fh(9, 0x65e3ed6682944d20L)); // ??1CTraceSnapshot@@QEAA@XZ + REMOVE_HASHES.add(fh(8, 0x6ce601a9094cc769L)); // destructor + REMOVE_HASHES.add(fh(7, 0x6e6e4f635fd59012L)); // destructor + REMOVE_HASHES.add(fh(8, 0xad4b1b8ef3775874L)); // pass derefed first param to subfunc + REMOVE_HASHES.add(fh(8, 0xba8d3a7590fcc497L)); // destructor + REMOVE_HASHES.add(fh(7, 0xc4312b6b7324334aL)); // destructor + REMOVE_HASHES.add(fh(11, 0xd6650c343cfb0baeL)); // constructor + REMOVE_HASHES.add(fh(13, 0xe6202628da545409L)); // ??0CXMLParserRoot@@QEAA@XZ + REMOVE_HASHES.add(fh(6, 0xebd9388fe0c5b3d5L)); // operator++ + REMOVE_HASHES.add(fh(8, 0xebfa00c4b493da85L)); // constructor + REMOVE_HASHES.add(fh(11, 0xf5d1f9d9b010f936L)); // constructor + REMOVE_HASHES.add(fh(11, 0xf7f34c91b43fa7d3L)); // destructor + REMOVE_HASHES.add(fh(8, 0xfcffed6fce4974a8L)); // constructor + REMOVE_HASHES.add(fh(9, 0x1c82d67f2be6ec3cL)); // constructor + REMOVE_HASHES.add(fh(10, 0x7dd643f9a75d75c4L)); // constructor + REMOVE_HASHES.add(fh(5, 0x88b326842c8ac560L)); // constructor + REMOVE_HASHES.add(fh(6, 0xa03ab775e8816d83L)); // wrapper + REMOVE_HASHES.add(fh(15, 0xe964369cf92d003fL)); // constructor + REMOVE_HASHES.add(fh(10, 0xf36c24d70ec93888L)); // wrapper + REMOVE_HASHES.add(fh(7, 0x763a8202f3d3c655L)); // constructor + REMOVE_HASHES.add(fh(12, 0x108d55ea8a0d124cL)); // scalar_deleting_destructor + REMOVE_HASHES.add(fh(13, 0x73f55a446deac3b1L)); // scalar_deleting_destructor + REMOVE_HASHES.add(fh(14, 0x3266cd569ab4eeffL)); // comparator + REMOVE_HASHES.add(fh(34, 0x8069e1fe2475e7c0L)); // inlined copies followed by method call + REMOVE_HASHES.add(fh(8, 0xa5c0d8da585783d3L)); // constructor + REMOVE_HASHES.add(fh(14, 0xad597a6e08b319b6L)); // destructor + REMOVE_HASHES.add(fh(17, 0xa552112bff535b06L)); // destructor + REMOVE_HASHES.add(fh(13, 0xac7036a5a6a27973L)); // destructor + REMOVE_HASHES.add(fh(6, 0x561ffc1c6cdb8a09L)); // Mysize + REMOVE_HASHES.add(fh(8, 0x6838c16db21b0fcdL)); // ??1_AsyncTaskCollection@details@Concurrency@@UEAA@XZ FORCE_RELATION.add(fh(6, 0x508d431b82512d5bL)); // Generic wrapper, one obvious child FORCE_RELATION.add(fh(19, 0x1e68c4d4d83e7585L)); // A little too generic stream thing, force parent @@ -275,12 +318,59 @@ public class RemoveFunctions extends GhidraScript { FORCE_RELATION.add(fh(25, 0x6a5c4f8adc931359L)); // scalar_deleting_destructor force parent FORCE_RELATION.add(fh(13, 0xf1e4167aedf569aL)); // Generic form, with many children FORCE_RELATION.add(fh(20, 0x678b611a60783c98L)); // Generic form with children + FORCE_RELATION.add(fh(15, 0x51980975b49f9f73L)); // ??1SchedulingNode@details@Concurrency@@QEAA@XZ + FORCE_RELATION.add(fh(18, 0xcf323a39c909432bL)); // ?_Future_error_map@std@@YAPEBDH@Z + FORCE_RELATION.add(fh(14, 0x41110421841870bdL)); // iterator::operator= FORCE_SPECIFIC.add(fh(26, 0xf0f7f2439683bfeaL)); // Variants with specialized constants FORCE_SPECIFIC.add(fh(17, 0xf468f6c40495d8caL)); // Dispatcher form with lots of specific constants FORCE_SPECIFIC.add(fh(13, 0x8779436db6c1d90L)); // ??1?$basic_string@GU?$char_traits@G@std@@V?$allocator@G@2@V_STL70@@@std@@QEAA@XZ FORCE_SPECIFIC.add(fh(75, 0x48156d182763009dL)); // _write confused with _read FORCE_SPECIFIC.add(fh(51, 0x5c02a83d7b53cabbL)); // ??1CCommandLineInfo@@UEAA@XZ + FORCE_SPECIFIC.add(fh(15, 0x4389c3585fa0606aL)); // has_flag + FORCE_SPECIFIC.add(fh(15, 0xcc72f3219032eacfL)); // ?__acrt_stdio_parse_mode_D@@YA_NAEAU__acrt_stdio_stream_mode@@@Z + FORCE_SPECIFIC.add(fh(36, 0xa07803de9bbbebbbL)); // vector deleting destructor + + FORCE_SPECIFIC.add(fh(12, 0x1997c3c57f1359d6L)); // ?dtor$9@?0??AddMenuCommands@CMFCToolBarsCustomizeDialog@@QEAAXPEBVCMenu@@HPEB_W1@Z@4HA + FORCE_RELATION.add(fh(12, 0x1997c3c57f1359d6L)); + FORCE_SPECIFIC.add(fh(11, 0x2c36a67a489920daL)); // ??1ContextBase@details@Concurrency@@UEAA@XZ + FORCE_RELATION.add(fh(11, 0x2c36a67a489920daL)); + FORCE_SPECIFIC.add(fh(15, 0x69156cfc34e915e7L)); // ??0@@QEAA@AEBV0@@Z + FORCE_RELATION.add(fh(15, 0x69156cfc34e915e7L)); + FORCE_SPECIFIC.add(fh(11, 0x781f34966e8ef7deL)); // FreeNode + FORCE_RELATION.add(fh(11, 0x781f34966e8ef7deL)); + FORCE_SPECIFIC.add(fh(11, 0x86b8837a96bb2fadL)); // ??0NumaInformation@SchedulerBase@details@Concurrency@@QEAA@XZ + FORCE_RELATION.add(fh(11, 0x86b8837a96bb2fadL)); + FORCE_SPECIFIC.add(fh(10, 0x87dd953d6c34861eL)); // get_allocator + FORCE_RELATION.add(fh(10, 0x87dd953d6c34861eL)); + FORCE_SPECIFIC.add(fh(14, 0x8c77a958beb620dfL)); // ?HasRealizedChores@ScheduleGroupSegmentBase@details@Concurrency@@IEBA_NXZ + FORCE_RELATION.add(fh(14, 0x8c77a958beb620dfL)); + FORCE_SPECIFIC.add(fh(10, 0x90181c81962ce711L)); // ??1CStreamOnCString@@QEAA@XZ + FORCE_RELATION.add(fh(10, 0x90181c81962ce711L)); + FORCE_SPECIFIC.add(fh(17, 0x9248e909511b8ddaL)); // ??0FreeVirtualProcessorRoot@details@Concurrency@@QEAA@PEAVSchedulerProxy@12@PEAUSchedulerNode@12@I@Z + FORCE_RELATION.add(fh(17, 0x9248e909511b8ddaL)); + FORCE_SPECIFIC.add(fh(15, 0x977e6c2c25e8f389L)); // ??1CMFCVisualManagerBitmapCache@@UEAA@XZ + FORCE_RELATION.add(fh(15, 0x977e6c2c25e8f389L)); + FORCE_SPECIFIC.add(fh(17, 0xad2d41b71db78df0L)); // ??1COleDispatchException@@UEAA@XZ + FORCE_RELATION.add(fh(17, 0xad2d41b71db78df0L)); + FORCE_SPECIFIC.add(fh(26, 0xb5b192b00955d1feL)); // ??1_Locinfo@std@@QEAA@XZ + FORCE_RELATION.add(fh(26, 0xb5b192b00955d1feL)); + FORCE_SPECIFIC.add(fh(10, 0xd6a5fc3691b7f101L)); // DelRegTree + FORCE_RELATION.add(fh(10, 0xd6a5fc3691b7f101L)); + FORCE_SPECIFIC.add(fh(13, 0xd75e6989aec2a5a9L)); // destructor form + FORCE_RELATION.add(fh(13, 0xd75e6989aec2a5a9L)); + FORCE_SPECIFIC.add(fh(13, 0xdcfdfd3a9345b3a5L)); // constructor form + FORCE_RELATION.add(fh(13, 0xdcfdfd3a9345b3a5L)); + FORCE_SPECIFIC.add(fh(14, 0xf323b038a8b540faL)); // ?GetPolicy@SchedulerBase@details@Concurrency@@UEBA?AVSchedulerPolicy@3@XZ + FORCE_RELATION.add(fh(14, 0xf323b038a8b540faL)); + FORCE_SPECIFIC.add(fh(10, 0x6862321b024d5c83L)); // constructor + FORCE_RELATION.add(fh(10, 0x6862321b024d5c83L)); + FORCE_SPECIFIC.add(fh(11, 0xa61ae8d54cfc35d6L)); // comparator, empty + FORCE_RELATION.add(fh(11, 0xa61ae8d54cfc35d6L)); + FORCE_SPECIFIC.add(fh(12, 0x892067d7b5484452L)); // anonymous destructor + FORCE_RELATION.add(fh(12, 0x892067d7b5484452L)); + FORCE_SPECIFIC.add(fh(12, 0x2155a28b83bb2704L)); // destructor + FORCE_RELATION.add(fh(12, 0x2155a28b83bb2704L)); FORCE_SPECIFIC.add(fh(10, 0x5c4a91ec77ecc3d2L)); // strnlen AUTO_PASS.add(fh(10, 0x5c4a91ec77ecc3d2L)); @@ -292,6 +382,16 @@ public class RemoveFunctions extends GhidraScript { AUTO_PASS.add(fh(10, 0xaba76591680821c6L)); FORCE_SPECIFIC.add(fh(10, 0x6244ea7ccad27b93L)); // wcsnlen AUTO_PASS.add(fh(10, 0x6244ea7ccad27b93L)); + FORCE_SPECIFIC.add(fh(9, 0x347bdce6848098d1L)); // strncnt + AUTO_PASS.add(fh(9, 0x347bdce6848098d1L)); + FORCE_SPECIFIC.add(fh(10, 0x668ac85a3d5bf04bL)); // raise_securityfailure + AUTO_PASS.add(fh(10, 0x668ac85a3d5bf04bL)); + + // Must exhibit relation and constants, plus one specific set of constants are marked as auto-fail + FORCE_SPECIFIC.add(fh(11, 0xf8ff33ae3bb6b9e9L)); // constructor + FORCE_RELATION.add(fh(11, 0xf8ff33ae3bb6b9e9L)); + REMOVE_SPECHASHES.add(fh(11, 0xc03985b32a1f76ceL)); + } private static Pair fh(int codeUnits, long digest) { @@ -403,6 +503,17 @@ public class RemoveFunctions extends GhidraScript { monitor.incrementProgress(1); } + monitor.setMaximum(REMOVE_SPECHASHES.size()); + monitor.setProgress(0); + for (Pair pair : REMOVE_SPECHASHES) { + List listSpecHash = + modifiableFidDB.findFunctionsBySpecificHash(pair.second.longValue()); + for (FunctionRecord funcRec : listSpecHash) { + modifiableFidDB.setAutoFailOnFunction(funcRec, true); + } + monitor.checkCanceled(); + monitor.incrementProgress(1); + } modifiableFidDB.saveDatabase("", monitor); } finally { diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java index 81ef79796a..8f6f04daa2 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/cmd/ApplyFidEntriesCommand.java @@ -18,6 +18,7 @@ package ghidra.feature.fid.cmd; import java.io.IOException; import java.util.*; +import ghidra.app.cmd.label.SetLabelPrimaryCmd; import ghidra.app.util.demangler.DemangledObject; import ghidra.feature.fid.db.FidQueryService; import ghidra.feature.fid.service.*; @@ -32,12 +33,16 @@ import ghidra.util.task.TaskMonitor; public class ApplyFidEntriesCommand extends BackgroundCommand { public static final String FID_CONFLICT = "FID_conflict:"; + public static final String FID_BOOKMARK_CATEGORY = "Function ID Analyzer"; + public static final String FIDCONFLICT_BOOKMARK_CATEGORY = "Function ID Conflict"; public static final int MAGIC_MULTIPLE_MATCH_LIMIT = 10; public static final int MAGIC_MULTIPLE_LIBRARY_LIMIT = 5; public static final int MAX_PLATE_COMMENT_LINE_LENGTH = 58; private MatchNameAnalysis nameAnalysis = new MatchNameAnalysis(); private AddressSet affectedLocations = new AddressSet(); + private TreeMap multiMatchNames = new TreeMap(); + private LinkedList
conflictFunctions = new LinkedList
(); private boolean alwaysApplyFidLabels; private float scoreThreshold; private float multiNameScoreThreshold; @@ -66,6 +71,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { try (FidQueryService fidQueryService = service.openFidQueryService(program.getLanguage(), false)) { + monitor.setMessage("FID Analysis"); List processProgram = service.processProgram(program, fidQueryService, scoreThreshold, monitor); if (processProgram == null) { @@ -88,6 +94,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { " at " + entry.function.getEntryPoint()); } } + applyConflictLabels(program); } catch (CancelledException e) { return false; @@ -143,8 +150,8 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { } // multiple matches - TODO: change to show classes vs libraries - libraries with same name don't put "base" name only for class ones - plateCommentContents = generateComment(plateCommentContents, true, false, monitor); - bookmarkContents = generateBookmark(bookmarkContents, true, false, monitor); + plateCommentContents = generateComment(plateCommentContents, monitor); + bookmarkContents = generateBookmark(bookmarkContents); applyMarkup(result.function, newFunctionName, plateCommentContents, bookmarkContents, monitor); @@ -155,20 +162,19 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { int counter = 0; - if (nameAnalysis.numNames() < MAGIC_MULTIPLE_MATCH_LIMIT) { - buffer.append("Name: "); - Iterator iterator = nameAnalysis.getNameIterator(); - while (iterator.hasNext()) { - monitor.checkCanceled(); - if (counter != 0) { - buffer.append(", "); - } - buffer.append(iterator.next()); - counter++; + Iterator iterator = nameAnalysis.getNameIterator(); + while (iterator.hasNext()) { + monitor.checkCanceled(); + buffer.append(' '); + buffer.append(iterator.next()); + buffer.append('\n'); + counter++; + if (counter > 3) { + break; } } - else { - buffer.append("Names: " + nameAnalysis.numSimilarNames() + " - too many to list"); + if (iterator.hasNext()) { + buffer.append(" " + nameAnalysis.numNames() + " names - too many to list\n"); } return buffer.toString(); @@ -203,8 +209,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { return buffer.toString(); } - private String generateComment(String header, boolean includeNames, boolean includeNamespaces, - TaskMonitor monitor) throws CancelledException { + private String generateComment(String header, TaskMonitor monitor) throws CancelledException { StringBuilder buffer = new StringBuilder(); buffer.append(header); @@ -217,18 +222,13 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { return buffer.toString(); } - private String generateBookmark(String bookmark, boolean includeNames, - boolean includeNamespaces, TaskMonitor monitor) throws CancelledException { + private String generateBookmark(String bookmark) { StringBuilder buffer = new StringBuilder(); if (createBookmarksEnabled) { buffer.append(bookmark); - // append names, class, and library info buffer buffer.append(" "); - buffer.append(listNames(monitor)); - - buffer.append(", "); - buffer.append(listLibraries(monitor)); + buffer.append(nameAnalysis.getNameIterator().next()); } return buffer.toString(); @@ -242,25 +242,22 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { return; } - int numUniqueLabelNames; - // single name case ok if (newFunctionName != null) { addFunctionLabel(function, newFunctionName, monitor); - numUniqueLabelNames = 1; } // multiple names else { - numUniqueLabelNames = addFunctionLabelMultipleMatches(function, monitor); + addFunctionLabelMultipleMatches(function, monitor); } - if (numUniqueLabelNames < MAGIC_MULTIPLE_MATCH_LIMIT) { - if (plateCommentContents != null && !plateCommentContents.equals("")) { - function.setComment(plateCommentContents); - } - if (bookmarkContents != null && !bookmarkContents.equals("")) { - function.getProgram().getBookmarkManager().setBookmark(function.getEntryPoint(), - BookmarkType.ANALYSIS, "Function ID Analyzer", bookmarkContents); - } + if (plateCommentContents != null && !plateCommentContents.equals("")) { + function.setComment(plateCommentContents); + } + if (bookmarkContents != null && !bookmarkContents.equals("")) { + function.getProgram() + .getBookmarkManager() + .setBookmark(function.getEntryPoint(), + BookmarkType.ANALYSIS, FID_BOOKMARK_CATEGORY, bookmarkContents); } } @@ -283,8 +280,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { return false; } - private void addFunctionLabel(Function function, String newFunctionName, TaskMonitor monitor) - throws CancelledException { + private void addFunctionLabel(Function function, String newFunctionName, TaskMonitor monitor) { removeConflictSymbols(function, newFunctionName, monitor); @@ -292,106 +288,64 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { addSymbolToFunction(function, newFunctionName); } - // This is called when a single library match is made. It checks to see if the label of the single match is contained in - // any "libID_conflict" labels. If it is, that label is removed from the other function(s) since it is no longer a possibility. - // Also checks those locations to see if there is only one other libID_conflict label left and if so, removes the "libID_conflict" - // prefix - private void removeConflictSymbols(Function function, String matchName, TaskMonitor monitor) - throws CancelledException { - - Program program = function.getProgram(); - SymbolTable symTab = program.getSymbolTable(); - - //get all currently created FID functions - BookmarkManager bkMgr = program.getBookmarkManager(); - Iterator bkmkIterator = bkMgr.getBookmarksIterator("Function ID Analyzer"); - - //iterate over all instances of libID_conflict__ and delete them - while (bkmkIterator.hasNext()) { - monitor.checkCanceled(); - - Bookmark nextBkmark = bkmkIterator.next(); - Symbol symbols[] = symTab.getSymbols(nextBkmark.getAddress()); - for (Symbol symbol : symbols) { - monitor.checkCanceled(); - - //if the symbol matches prefix+matchName exactly - //OR if no _ exists immediately after the prefix + matchName part of the found symbol (meaning there is no address following) - //OR if there is an _ immediately following the prefix+Name AND and address directly following it we can be sure it is a good symbol match and can delete it - //otherwise it contains extra characters indicating an invalid match and we don't want to delete it - String name = symbol.getName(); - - if (name.startsWith(FID_CONFLICT)) { - String baseName = name.substring(FID_CONFLICT.length()); - Address symAddr = symbol.getAddress(); - if (baseName.equals(matchName)) { - - symbol.delete(); - - // check to see if there is only one symbol left on that function - // that has a libID_conflict name and if so remove the conflict part - removeConflictFromSymbolWhenOnlyOneLeft(symTab, symAddr, monitor); - break; + /** + * Delete a symbol of the given name and address, knowing there are multiple Symbols at the address. + * If the symbol is primary, make another Symbol at the address primary before deleting + * @param matchName is the given Symbol name + * @param addr is the given Address + * @param program is the Program + * @return the number of Symbols remaining at the address + */ + private int deleteSymbol(String matchName, Address addr, Program program) { + int numSymbols = 0; + for (int i = 0; i < 2; ++i) { // Try to find non-primary matching Symbol at most twice + Symbol[] symbols = program.getSymbolTable().getSymbols(addr); + numSymbols = symbols.length; + if (numSymbols <= 1) { + break; + } + for (Symbol sym : symbols) { // Among Symbols at the Address + if (sym.getName().equals(matchName)) { // Find one with matching name + if (!sym.isPrimary()) { // If it is not primary + sym.delete(); // delete it immediately + numSymbols -= 1; + break; // and we are done } - } - } - } - } - - //check to see if there is only one symbol left on that function - // that has a libID_conflict name and if so remove the conflict part - private void removeConflictFromSymbolWhenOnlyOneLeft(SymbolTable symTab, Address symAddr, - TaskMonitor monitor) throws CancelledException { - - //First get all symbols at the removed symbol address - Symbol[] symbols = symTab.getSymbols(symAddr); - - //Next, check to see if there is only one symbol, if it has prefix remove the prefix, otherwise just skip to end - if (symbols.length == 1) { - if (symbols[0].getName().startsWith(FID_CONFLICT)) { - removeConflictFromSymbol(symbols[0].getSource(), symbols[0]); - } - } - - // if more than one symbol, check to see if only one has libIDconflict - else { - int conflictCount = 0; - Symbol keepSymbol = null; - SourceType keepSymbolSource = null; - - for (Symbol symbol : symbols) { - monitor.checkCanceled(); - - if (conflictCount > 1) { - keepSymbol = null; - keepSymbolSource = null; + Symbol otherSym = symbols[0]; + if (otherSym == sym) { // Otherwise find another Symbol, which must not be primary + otherSym = symbols[1]; + } + // Set the other symbol to primary + SetLabelPrimaryCmd cmd = new SetLabelPrimaryCmd(addr, otherSym.getName(), + otherSym.getParentNamespace()); + cmd.applyTo(program); break; } - if (symbol.getName().startsWith(FID_CONFLICT)) { - conflictCount++; - keepSymbol = symbol; - keepSymbolSource = symbol.getSource(); - } } - if (keepSymbol != null) { - removeConflictFromSymbol(keepSymbolSource, keepSymbol); - } - } + return numSymbols; } - private void removeConflictFromSymbol(SourceType sourceType, Symbol symbol) { + // This is called when a single library match is made. It checks to see if the label of the single match is contained in + // the set of "FID conflict" functions with multiple matches. + // If it is, that label is removed from the other function(s) since it is no longer a possibility. + // Also checks those locations to see if there is only one label left and if so, removes the "FID conflict" bookmark. + private void removeConflictSymbols(Function function, String matchName, TaskMonitor monitor) { - String newName = symbol.getName().substring(FID_CONFLICT.length()); - try { - symbol.setName(newName, sourceType); + Address addr = multiMatchNames.get(matchName); + if (addr == null) { + return; } - catch (DuplicateNameException e) { - Msg.warn(SymbolUtilities.class, - "Duplicate symbol name \"" + newName + "\" at " + symbol.getAddress()); - } - catch (InvalidInputException e) { - throw new AssertException(e); // unexpected + Program program = function.getProgram(); + int numSymbols = deleteSymbol(matchName, addr, program); + if (numSymbols <= 1) { + // Only one symbol left, delete the "FID conflict" bookmark + BookmarkManager bookmarkManager = program.getBookmarkManager(); + Bookmark bookmark = bookmarkManager.getBookmark(addr, BookmarkType.ANALYSIS, + FIDCONFLICT_BOOKMARK_CATEGORY); + if (bookmark != null) { + bookmarkManager.removeBookmark(bookmark); + } } } @@ -399,51 +353,93 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { throws CancelledException { Program program = function.getProgram(); - Set matchNames = nameAnalysis.getAppriateNamesSet(); - if (matchNames.size() >= MAGIC_MULTIPLE_MATCH_LIMIT) { - return matchNames.size(); + if (nameAnalysis.numNames() >= MAGIC_MULTIPLE_MATCH_LIMIT) { + return nameAnalysis.numNames(); } - Set unusedNames = getFIDNamesThatDontExistSomewhereElse(program, matchNames); + Symbol symbol = function.getSymbol(); + boolean preexistingSymbol = (symbol != null && symbol.getSource() != SourceType.DEFAULT); - for (String baseName : unusedNames) { + Set unusedNames = + getFIDNamesThatDontExistSomewhereElse(program, nameAnalysis.getNameIterator()); + + Address addr = function.getEntryPoint(); + for (String functionName : unusedNames) { monitor.checkCanceled(); - String functionName = getFunctionNameForBaseName(program, baseName, unusedNames); addSymbolToFunction(function, functionName); + multiMatchNames.put(functionName, addr); } + if (unusedNames.size() > 1) { + if (!preexistingSymbol) { + conflictFunctions.add(addr); + } + if (createBookmarksEnabled) { + BookmarkManager bookmarkManager = function.getProgram().getBookmarkManager(); + bookmarkManager.setBookmark(addr, BookmarkType.ANALYSIS, + FIDCONFLICT_BOOKMARK_CATEGORY, + "Multiple likely matching functions"); + } + } return unusedNames.size(); } /** - * Returns the symbol name to use based on if there are multiple conficts for a function - * If there is only one matching name, then it is used directly. Otherwise, "FID_conflict:" - * is prepended to the name. + * Apply special FID_CONFLICT to the primary symbol on functions where we had multiple matches + * @param program is the Program */ - private String getFunctionNameForBaseName(Program program, String baseName, - Set unusedNames) { - if (unusedNames.size() == 1) { - return baseName; + private void applyConflictLabels(Program program) { + SymbolTable symbolTable = program.getSymbolTable(); + for (Address addr : conflictFunctions) { + Symbol[] symbols = symbolTable.getSymbols(addr); + if (symbols.length <= 1) { + continue; // Only apply conflict label if more than one symbol at address + } + Symbol symbol = null; + for (Symbol symbol2 : symbols) { + if (symbol2.isPrimary()) { + symbol = symbol2; + break; + } + } + if (symbol == null || !symbol.isGlobal()) { + continue; + } + String baseName = symbol.getName(); + if (baseName.startsWith(FID_CONFLICT)) { + continue; // Conflict label previously applied + } + DemangledObject demangle = NameVersions.demangle(program, baseName); + if (demangle != null) { + baseName = demangle.getName(); + } + baseName = FID_CONFLICT + baseName; + try { + symbol = symbolTable.createLabel(addr, baseName, null, SourceType.ANALYSIS); + SetLabelPrimaryCmd cmd = + new SetLabelPrimaryCmd(addr, symbol.getName(), symbol.getParentNamespace()); + cmd.applyTo(program); + } + catch (InvalidInputException e) { + Msg.warn(SymbolUtilities.class, + "Invalid symbol name: \"" + baseName + "\" at " + addr); + } } - - DemangledObject demangledObj = NameVersions.demangle(program, baseName); - if (demangledObj != null) { - baseName = demangledObj.getName(); - } - return FID_CONFLICT + baseName; } /** * Takes a set of FID matching names and returns a subset that includes only names that don't exist * somewhere else in the program. */ - private Set getFIDNamesThatDontExistSomewhereElse(Program program, - Set matchNames) { + private static Set getFIDNamesThatDontExistSomewhereElse(Program program, + Iterator iter) { Set unusedNames = new HashSet(); - for (String name : matchNames) { - if (!nameExistsSomewhereElse(program.getSymbolTable(), name)) { + SymbolTable symbolTable = program.getSymbolTable(); + while (iter.hasNext()) { + String name = iter.next(); + if (!nameExistsSomewhereElse(symbolTable, name)) { unusedNames.add(name); } } @@ -451,7 +447,7 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { } //Check to see if other functions exist with the same baseName or _baseName or __baseName - private boolean nameExistsSomewhereElse(SymbolTable symTab, String baseName) { + private static boolean nameExistsSomewhereElse(SymbolTable symTab, String baseName) { //I did it this way because doing it with an iterator and wildcard was really really slow List globalSymbols = symTab.getLabelOrFunctionSymbols(baseName, null); @@ -470,7 +466,6 @@ public class ApplyFidEntriesCommand extends BackgroundCommand { } return false; - } private void addSymbolToFunction(Function function, String name) { diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidProgramSeeker.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidProgramSeeker.java index f417cdecf7..4157c1d857 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidProgramSeeker.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/FidProgramSeeker.java @@ -342,6 +342,10 @@ public class FidProgramSeeker { } } + if (functionRecord.isForceRelation() && childCodeUnits == 0) { + return null; + } + int parentCodeUnits = 0; if (family.getParents().size() < MAX_NUM_PARENTS_FOR_SCORE) { @@ -355,12 +359,6 @@ public class FidProgramSeeker { float functionScore = functionCodeUnits; functionScore += 0.67 * specificCodeUnits; // Each specific constant count is worth 2/3 of a whole code unit - if (functionRecord.isForceRelation()) { - // If both auto-pass and force-relation are on, do not ding score if some children match - if (!functionRecord.autoPass() || (childCodeUnits == 0)) { - functionScore = 0; - } - } float childScore = childCodeUnits; float parentScore = parentCodeUnits; if (functionScore + childScore + parentScore < scoreThreshold) { diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/MatchNameAnalysis.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/MatchNameAnalysis.java index 32d4340826..f36aef1e3c 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/MatchNameAnalysis.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/MatchNameAnalysis.java @@ -47,6 +47,10 @@ public class MatchNameAnalysis { return rawNames.iterator(); } + public boolean containsRawName(String name) { + return rawNames.contains(name); + } + public Iterator getNameIterator() { return finalNameList.iterator(); } @@ -89,10 +93,12 @@ public class MatchNameAnalysis { if (nameVersions.rawName != null) { rawNames.add(nameVersions.rawName); // Dedup the raw names similarBaseNames.add(nameVersions.similarName); // Dedup names with underscores removed - if (nameVersions.demangledBaseName != null) // If we can demangle + if (nameVersions.demangledBaseName != null) { exactDemangledBaseNames.add(nameVersions.demangledBaseName); // Dedup demangled base name - else + } + else { cannotDemangle += 1; + } } } @@ -117,6 +123,12 @@ public class MatchNameAnalysis { mostOptimisticCount = 1; finalNameList = Collections.singleton(singleName); } + else if (rawNames.size() > similarBaseNames.size()) { + // if names are the same except for underscores use the similar name + // list to remove dupes + finalNameList = similarBaseNames; + } + if (matches.size() > 0) { overallScore = matches.get(0).getOverallScore(); } @@ -132,8 +144,9 @@ public class MatchNameAnalysis { if (library != null) { libraries.add(library); } - if (libraries.size() >= libraryLimit) + if (libraries.size() >= libraryLimit) { break; + } } if (libraries.size() >= libraryLimit) { // Too many libraries to directly display // Try getting rid of the variant field, to see if we can reduce the count @@ -164,8 +177,9 @@ public class MatchNameAnalysis { } private String findCommonBaseName() { - if (similarBaseNames.size() == 1) + if (similarBaseNames.size() == 1) { return rawNames.iterator().next(); + } return null; } @@ -223,25 +237,4 @@ public class MatchNameAnalysis { } return finalName; } - - /** - * returns the appropriate set of names to use for laying down fid symbols - */ - public Set getAppriateNamesSet() { - // if the names were originally mangled and they demangle to more than - // one base name use the demangled base names - if (exactDemangledBaseNames.size() > 1 && rawNames.size() > 1) { - return exactDemangledBaseNames; - } - - // if names are the same except for underscores use the similar name - // list to remove dupes - if (rawNames.size() == similarBaseNames.size() * 2) { - return similarBaseNames; - } - - // else the names are not the same or similar so use the list of exact - // names - return rawNames; - } } diff --git a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/NameVersions.java b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/NameVersions.java index 4f84bd9ac2..4a4225eaed 100644 --- a/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/NameVersions.java +++ b/Ghidra/Features/FunctionID/src/main/java/ghidra/feature/fid/service/NameVersions.java @@ -15,8 +15,7 @@ */ package ghidra.feature.fid.service; -import ghidra.app.util.demangler.DemangledObject; -import ghidra.app.util.demangler.DemanglerUtil; +import ghidra.app.util.demangler.*; import ghidra.program.model.listing.Program; public class NameVersions { @@ -44,12 +43,51 @@ public class NameVersions { return null; } + private static String getBaseClass(Demangled namespace) { + String name = namespace.getNamespaceName(); // First level of namespace + // Check for evidence of anonymous or unnamed namespace, which won't be distinguishing + if (name.length() > 1 && name.charAt(0) == '`') { + char firstChar = name.charAt(1); + if (firstChar >= '0' && firstChar <= '9') { + return namespace.getNamespaceString(); // Get full namespace + } + } + return name; + } + + private static String constructBaseName(DemangledObject demangledObj) { + String origName = demangledObj.getName(); + String name = origName.replaceFirst("_*", ""); + Demangled namespace = demangledObj.getNamespace(); + if (namespace != null) { + if (name.endsWith("destructor'") || + name.startsWith("operator") || + name.startsWith("dtor$")) { + String baseClassName = getBaseClass(namespace); + if (baseClassName == null) { + return null; + } + return baseClassName + "::" + origName; + } + String fullString = namespace.getNamespaceString(); + if (fullString != null && fullString.startsWith("std::")) { + // Common containers, make sure we keep the whole name + if (fullString.startsWith("std::vector") || fullString.startsWith("std::list") || + fullString.startsWith("std::map") || fullString.startsWith("std::set") || + fullString.startsWith("std::basic_string")) { + return fullString + "::" + origName; + } + } + } + return name; + } + public static NameVersions generate(String rawName,Program program) { NameVersions result = new NameVersions(rawName); if (rawName != null) { DemangledObject demangledObj = demangle(program, rawName); if (demangledObj != null) { - result.demangledBaseName = demangledObj.getName().replaceFirst("_*", ""); + result.demangledBaseName = constructBaseName(demangledObj); } // Put base names with underscores removed in a HashSet