From 7aa6cdce9fa66dec367f8b4c1e3f80a0cd7d9cc2 Mon Sep 17 00:00:00 2001 From: dev747368 <48332326+dev747368@users.noreply.github.com> Date: Wed, 20 Jan 2021 20:09:54 -0500 Subject: [PATCH] GP-419 Improve speed of DWARF import --- .../util/bin/format/dwarf4/DebugInfoEntry.java | 18 ++++++++++++++++-- .../next/DWARFDataTypeConflictHandler.java | 7 +++++-- .../dwarf4/next/DWARFDataTypeImporter.java | 9 +++++++++ .../dwarf4/next/DWARFDataTypeManager.java | 9 +++++---- .../bin/format/dwarf4/next/DWARFParser.java | 15 +++++++++++++++ .../bin/format/dwarf4/next/DWARFProgram.java | 3 +-- 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DebugInfoEntry.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DebugInfoEntry.java index 9c5e885253..1e7c5059df 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DebugInfoEntry.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DebugInfoEntry.java @@ -19,8 +19,7 @@ import java.io.IOException; import java.util.*; import ghidra.app.util.bin.BinaryReader; -import ghidra.app.util.bin.format.dwarf4.attribs.DWARFAttributeFactory; -import ghidra.app.util.bin.format.dwarf4.attribs.DWARFAttributeValue; +import ghidra.app.util.bin.format.dwarf4.attribs.*; import ghidra.app.util.bin.format.dwarf4.encoding.DWARFAttribute; import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag; @@ -34,6 +33,14 @@ import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag; */ public class DebugInfoEntry { + /** + * List of common DWARF attributes that are not used currently in Ghidra. These attributes values will be + * thrown away during reading to save some memory. There are lots of attributes that Ghidra doesn't + * currently use, but they do not appear frequently enough to consume a significant amount of memory. + */ + private static final Set ATTRIBUTES_TO_SKIP = + Set.of(DWARFAttribute.DW_AT_sibling, DWARFAttribute.DW_AT_accessibility); + private final DWARFCompilationUnit compilationUnit; private final long offset; private final DWARFAbbreviation abbreviation; @@ -73,6 +80,13 @@ public class DebugInfoEntry { DWARFAttributeSpecification attributeSpec = attributeSpecs[i]; result.attributes[i] = attributeFactory.read(reader, unit, attributeSpec.getAttributeForm()); + + if (ATTRIBUTES_TO_SKIP.contains(attributeSpec.getAttribute())) { + // throw away the object holding the value and replace it with + // the static boolean true value object to hold its place in + // the list. This saves a little memory + result.attributes[i] = DWARFBooleanAttribute.TRUE; + } } return result; diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeConflictHandler.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeConflictHandler.java index a87b6c6969..b3e783f8d8 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeConflictHandler.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeConflictHandler.java @@ -20,6 +20,7 @@ import static ghidra.program.model.data.DataTypeConflictHandler.ConflictResult.* import java.util.*; import ghidra.program.model.data.*; +import ghidra.util.SystemUtilities; /** * This {@link DataTypeConflictHandler conflict handler} attempts to match @@ -167,7 +168,8 @@ class DWARFDataTypeConflictHandler extends DataTypeConflictHandler { DataTypeComponent fullDTCAt = (partDTC.getDataType() instanceof BitFieldDataType) ? getBitfieldByOffsets(full, partDTC) : full.getComponentAt(partDTC.getOffset()); - if (fullDTCAt == null || fullDTCAt.getOffset() != partDTC.getOffset()) { + if (fullDTCAt == null || fullDTCAt.getOffset() != partDTC.getOffset() || + !SystemUtilities.isEqual(fullDTCAt.getFieldName(), partDTC.getFieldName())) { return false; } DataType partDT = partDTC.getDataType(); @@ -218,7 +220,8 @@ class DWARFDataTypeConflictHandler extends DataTypeConflictHandler { */ private ConflictResult doStrictCompare(DataType addedDataType, DataType existingDataType, Set visitedDataTypes) { - if (!addVisited(existingDataType, addedDataType, visitedDataTypes)) { + if (addedDataType == existingDataType || + !addVisited(existingDataType, addedDataType, visitedDataTypes)) { return USE_EXISTING; } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeImporter.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeImporter.java index 42636e1507..724f5083c1 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeImporter.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeImporter.java @@ -26,6 +26,7 @@ import ghidra.app.util.DataTypeNamingUtil; import ghidra.app.util.bin.format.dwarf4.*; import ghidra.app.util.bin.format.dwarf4.encoding.*; import ghidra.app.util.bin.format.dwarf4.expression.DWARFExpressionException; +import ghidra.program.database.DatabaseObject; import ghidra.program.database.data.DataTypeUtilities; import ghidra.program.model.data.*; import ghidra.program.model.data.Enum; @@ -139,6 +140,10 @@ public class DWARFDataTypeImporter { if (result != null) { return result; } + DataType alreadyImportedDT = dwarfDTM.getDataType(diea.getOffset(), null); + if (alreadyImportedDT != null) { + return new DWARFDataType(alreadyImportedDT, null, diea.getOffset()); + } if (!trackRecursion(diea.getOffset(), 1)) { return defaultValue; @@ -229,6 +234,10 @@ public class DWARFDataTypeImporter { * offset -> ddt */ private void recordTempDataType(DWARFDataType ddt) { + if (ddt.dataType instanceof DatabaseObject) { + // don't store info about types that are already in the database + return; + } dataTypeInstanceToDDTMap.put(ddt.dataType, ddt); for (Long offset : ddt.offsets) { dieOffsetToDataTypeMap.put(offset, ddt); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeManager.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeManager.java index be2740b486..a420e6f3ba 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeManager.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeManager.java @@ -26,9 +26,10 @@ import ghidra.app.util.bin.format.dwarf4.next.DWARFDataTypeImporter.DWARFDataTyp import ghidra.program.model.data.*; import ghidra.program.model.listing.Program; import ghidra.util.Msg; -import ghidra.util.SystemUtilities; +import ghidra.util.Swing; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; +import utility.function.Dummy; /** * Manages mappings between DWARF DIEs and Ghidra DataTypes. @@ -117,9 +118,7 @@ public class DWARFDataTypeManager { // Wait for the swing thread to clear its event queue because we are running into // issues with the number of events overwhelming the swing thread. // This does slow us down a little bit but this makes the GUI responsive to the user. - SystemUtilities.runSwingNow(() -> { - /* nada */ - }); + Swing.runNow(Dummy.runnable()); DWARFDataTypeImporter ddtImporter = new DWARFDataTypeImporter(prog, this, prog.getImportOptions()); @@ -651,6 +650,8 @@ public class DWARFDataTypeManager { // parent's source info (handles auto-generated ctors and such) addDataType(diea.getOffset(), funcDefDT, DWARFSourceInfo.getSourceInfoWithFallbackToParent(diea)); + + Swing.runNow(Dummy.runnable()); } } } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFParser.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFParser.java index fed00d12b1..878e5b5746 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFParser.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFParser.java @@ -16,15 +16,18 @@ package ghidra.app.util.bin.format.dwarf4.next; import java.io.IOException; +import java.util.Collections; import java.util.List; import ghidra.app.plugin.core.datamgr.util.DataTypeUtils; import ghidra.app.util.bin.format.dwarf4.DWARFException; import ghidra.program.model.data.*; import ghidra.util.Msg; +import ghidra.util.Swing; import ghidra.util.exception.CancelledException; import ghidra.util.exception.DuplicateNameException; import ghidra.util.task.TaskMonitor; +import utility.function.Dummy; /** * Performs a DWARF datatype import and a DWARF function import, under the control of the @@ -63,7 +66,14 @@ public class DWARFParser { */ private void moveTypesIntoSourceFolders() throws CancelledException { + // Sort by category to reduce the amount of thrashing the DTM does reloading + // categories. List importedTypes = dwarfDTM.getImportedTypes(); + Collections.sort(importedTypes, + (dtp1, dtp2) -> dtp1.getCategoryPath() + .getPath() + .compareTo(dtp2.getCategoryPath().getPath())); + monitor.setIndeterminate(false); monitor.setShowProgressValue(true); monitor.initialize(importedTypes.size()); @@ -76,6 +86,11 @@ public class DWARFParser { monitor.checkCanceled(); monitor.incrementProgress(1); + if ( (monitor.getProgress() % 5) == 0 ) { + /* balance between getting work done and pampering the swing thread */ + Swing.runNow(Dummy.runnable()); + } + DataType dataType = prog.getGhidraProgram().getDataTypeManager().getDataType(dataTypePath); if (dataType != null && !(dataType instanceof Pointer || dataType instanceof Array)) { diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFProgram.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFProgram.java index c5bf4f418b..6462efea8e 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFProgram.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFProgram.java @@ -770,8 +770,7 @@ public class DWARFProgram implements Closeable { if (refdOffset == -1) { continue; } - DWARFCompilationUnit targetCU = getCompilationUnitFor(refdOffset); - if (targetCU != null && targetCU != die.getCompilationUnit()) { + if (!die.getCompilationUnit().containsOffset(refdOffset)) { return true; } }