Merge remote-tracking branch 'origin/GP-419_dev747368_dwarf_speed_improvements--SQUASHED'

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/DebugInfoEntry.java

@@ -19,8 +19,7 @@ import java.io.IOException;
 import java.util.*;
 
 import ghidra.app.util.bin.BinaryReader;
-import ghidra.app.util.bin.format.dwarf4.attribs.DWARFAttributeFactory;
-import ghidra.app.util.bin.format.dwarf4.attribs.DWARFAttributeValue;
+import ghidra.app.util.bin.format.dwarf4.attribs.*;
 import ghidra.app.util.bin.format.dwarf4.encoding.DWARFAttribute;
 import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag;
 
@@ -34,6 +33,14 @@ import ghidra.app.util.bin.format.dwarf4.encoding.DWARFTag;
  */
 public class DebugInfoEntry {
 
+	/**
+	 * List of common DWARF attributes that are not used currently in Ghidra.  These attributes values will be
+	 * thrown away during reading to save some memory.  There are lots of attributes that Ghidra doesn't
+	 * currently use, but they do not appear frequently enough to consume a significant amount of memory.
+	 */
+	private static final Set<Integer> ATTRIBUTES_TO_SKIP =
+		Set.of(DWARFAttribute.DW_AT_sibling, DWARFAttribute.DW_AT_accessibility);
+
 	private final DWARFCompilationUnit compilationUnit;
 	private final long offset;
 	private final DWARFAbbreviation abbreviation;
@@ -73,6 +80,13 @@ public class DebugInfoEntry {
 			DWARFAttributeSpecification attributeSpec = attributeSpecs[i];
 			result.attributes[i] =
 				attributeFactory.read(reader, unit, attributeSpec.getAttributeForm());
+
+			if (ATTRIBUTES_TO_SKIP.contains(attributeSpec.getAttribute())) {
+				// throw away the object holding the value and replace it with
+				// the static boolean true value object to hold its place in
+				// the list.  This saves a little memory
+				result.attributes[i] = DWARFBooleanAttribute.TRUE;
+			}
 		}
 
 		return result;

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeConflictHandler.java

@@ -20,6 +20,7 @@ import static ghidra.program.model.data.DataTypeConflictHandler.ConflictResult.*
 import java.util.*;
 
 import ghidra.program.model.data.*;
+import ghidra.util.SystemUtilities;
 
 /**
  * This {@link DataTypeConflictHandler conflict handler} attempts to match
@@ -167,7 +168,8 @@ class DWARFDataTypeConflictHandler extends DataTypeConflictHandler {
 			DataTypeComponent fullDTCAt = (partDTC.getDataType() instanceof BitFieldDataType)
 					? getBitfieldByOffsets(full, partDTC)
 					: full.getComponentAt(partDTC.getOffset());
-			if (fullDTCAt == null || fullDTCAt.getOffset() != partDTC.getOffset()) {
+			if (fullDTCAt == null || fullDTCAt.getOffset() != partDTC.getOffset() ||
+				!SystemUtilities.isEqual(fullDTCAt.getFieldName(), partDTC.getFieldName())) {
 				return false;
 			}
 			DataType partDT = partDTC.getDataType();
@@ -218,7 +220,8 @@ class DWARFDataTypeConflictHandler extends DataTypeConflictHandler {
 	 */
 	private ConflictResult doStrictCompare(DataType addedDataType, DataType existingDataType,
 			Set<Long> visitedDataTypes) {
-		if (!addVisited(existingDataType, addedDataType, visitedDataTypes)) {
+		if (addedDataType == existingDataType ||
+			!addVisited(existingDataType, addedDataType, visitedDataTypes)) {
 			return USE_EXISTING;
 		}
 

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeImporter.java

@@ -26,6 +26,7 @@ import ghidra.app.util.DataTypeNamingUtil;
 import ghidra.app.util.bin.format.dwarf4.*;
 import ghidra.app.util.bin.format.dwarf4.encoding.*;
 import ghidra.app.util.bin.format.dwarf4.expression.DWARFExpressionException;
+import ghidra.program.database.DatabaseObject;
 import ghidra.program.database.data.DataTypeUtilities;
 import ghidra.program.model.data.*;
 import ghidra.program.model.data.Enum;
@@ -139,6 +140,10 @@ public class DWARFDataTypeImporter {
 		if (result != null) {
 			return result;
 		}
+		DataType alreadyImportedDT = dwarfDTM.getDataType(diea.getOffset(), null);
+		if (alreadyImportedDT != null) {
+			return new DWARFDataType(alreadyImportedDT, null, diea.getOffset());
+		}
 
 		if (!trackRecursion(diea.getOffset(), 1)) {
 			return defaultValue;
@@ -229,6 +234,10 @@ public class DWARFDataTypeImporter {
 	 * 						offset -> ddt
 	 */
 	private void recordTempDataType(DWARFDataType ddt) {
+		if (ddt.dataType instanceof DatabaseObject) {
+			// don't store info about types that are already in the database
+			return;
+		}
 		dataTypeInstanceToDDTMap.put(ddt.dataType, ddt);
 		for (Long offset : ddt.offsets) {
 			dieOffsetToDataTypeMap.put(offset, ddt);

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFDataTypeManager.java

@@ -26,9 +26,10 @@ import ghidra.app.util.bin.format.dwarf4.next.DWARFDataTypeImporter.DWARFDataTyp
 import ghidra.program.model.data.*;
 import ghidra.program.model.listing.Program;
 import ghidra.util.Msg;
-import ghidra.util.SystemUtilities;
+import ghidra.util.Swing;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
+import utility.function.Dummy;
 
 /**
  * Manages mappings between DWARF DIEs and Ghidra DataTypes.
@@ -117,9 +118,7 @@ public class DWARFDataTypeManager {
 		// Wait for the swing thread to clear its event queue because we are running into
 		// issues with the number of events overwhelming the swing thread.
 		// This does slow us down a little bit but this makes the GUI responsive to the user.
-		SystemUtilities.runSwingNow(() -> {
-			/* nada */
-		});
+		Swing.runNow(Dummy.runnable());
 
 		DWARFDataTypeImporter ddtImporter =
 			new DWARFDataTypeImporter(prog, this, prog.getImportOptions());
@@ -651,6 +650,8 @@ public class DWARFDataTypeManager {
 						// parent's source info (handles auto-generated ctors and such)
 						addDataType(diea.getOffset(), funcDefDT,
 							DWARFSourceInfo.getSourceInfoWithFallbackToParent(diea));
+
+						Swing.runNow(Dummy.runnable());
 					}
 				}
 			}

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFParser.java

@@ -16,15 +16,18 @@
 package ghidra.app.util.bin.format.dwarf4.next;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 
 import ghidra.app.plugin.core.datamgr.util.DataTypeUtils;
 import ghidra.app.util.bin.format.dwarf4.DWARFException;
 import ghidra.program.model.data.*;
 import ghidra.util.Msg;
+import ghidra.util.Swing;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.exception.DuplicateNameException;
 import ghidra.util.task.TaskMonitor;
+import utility.function.Dummy;
 
 /**
  * Performs a DWARF datatype import and a DWARF function import, under the control of the
@@ -63,7 +66,14 @@ public class DWARFParser {
 	 */
 	private void moveTypesIntoSourceFolders() throws CancelledException {
 
+		// Sort by category to reduce the amount of thrashing the DTM does reloading
+		// categories.
 		List<DataTypePath> importedTypes = dwarfDTM.getImportedTypes();
+		Collections.sort(importedTypes,
+			(dtp1, dtp2) -> dtp1.getCategoryPath()
+					.getPath()
+					.compareTo(dtp2.getCategoryPath().getPath()));
+
 		monitor.setIndeterminate(false);
 		monitor.setShowProgressValue(true);
 		monitor.initialize(importedTypes.size());
@@ -76,6 +86,11 @@ public class DWARFParser {
 			monitor.checkCanceled();
 			monitor.incrementProgress(1);
 
+			if ( (monitor.getProgress() % 5) == 0 ) {
+				/* balance between getting work done and pampering the swing thread */ 
+				Swing.runNow(Dummy.runnable());
+			}
+
 			DataType dataType =
 				prog.getGhidraProgram().getDataTypeManager().getDataType(dataTypePath);
 			if (dataType != null && !(dataType instanceof Pointer || dataType instanceof Array)) {

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf4/next/DWARFProgram.java

@@ -770,8 +770,7 @@ public class DWARFProgram implements Closeable {
 				if (refdOffset == -1) {
 					continue;
 				}
-				DWARFCompilationUnit targetCU = getCompilationUnitFor(refdOffset);
-				if (targetCU != null && targetCU != die.getCompilationUnit()) {
+				if (!die.getCompilationUnit().containsOffset(refdOffset)) {
 					return true;
 				}
 			}