Merge remote-tracking branch 'origin/GP-5221_dwarf_debug_relative_paths--SQUASHED' into Ghidra_11.3

Ghidra/Features/Base/ghidra_scripts/DWARFLineInfoSourceMapScript.java

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 // Adds DWARF source file line number info to the current program as source map entries.
+// A source file that is relative after path normalization will have all leading "."
+// and "/../" entries stripped and then be placed under an artificial directory.
 // Note that you can run this script on a program that has already been analyzed by the
 // DWARF analyzer.
 //@category DWARF
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 import ghidra.app.script.GhidraScript;
 import ghidra.app.util.bin.BinaryReader;
@@ -30,14 +31,19 @@ import ghidra.app.util.bin.format.dwarf.sectionprovider.DWARFSectionProviderFact
 import ghidra.framework.store.LockException;
 import ghidra.program.database.sourcemap.SourceFile;
 import ghidra.program.database.sourcemap.SourceFileIdType;
-import ghidra.program.model.address.Address;
-import ghidra.program.model.address.AddressOverflowException;
+import ghidra.program.model.address.*;
 import ghidra.program.model.sourcemap.SourceFileManager;
 import ghidra.util.Msg;
+import ghidra.util.SourceFileUtils;
 import ghidra.util.exception.CancelledException;
 
 public class DWARFLineInfoSourceMapScript extends GhidraScript {
 	public static final int ENTRY_MAX_LENGTH = 1000;
+	private static final int MAX_ERROR_MSGS_TO_DISPLAY = 25;
+	private static final int MAX_WARNING_MSGS_TO_DISPLAY = 25;
+	private static final String COMPILATION_ROOT_DIRECTORY = DWARFImporter.DEFAULT_COMPILATION_DIR;
+	private int numErrors;
+	private int numWarnings;
 
 	@Override
 	protected void run() throws Exception {
@@ -69,71 +75,111 @@ public class DWARFLineInfoSourceMapScript extends GhidraScript {
 			return;
 		}
 		int entryCount = 0;
-		monitor.initialize(reader.length(), "DWARF Source Map Info");
 		List<DWARFCompilationUnit> compUnits = dprog.getCompilationUnits();
 		SourceFileManager sourceManager = currentProgram.getSourceFileManager();
 		List<SourceFileAddr> sourceInfo = new ArrayList<>();
+		monitor.initialize(compUnits.size(), "DWARF: Reading Source Map Info");
 		for (DWARFCompilationUnit cu : compUnits) {
+			monitor.increment();
 			sourceInfo.addAll(cu.getLine().getAllSourceFileAddrInfo(cu, reader));
 		}
+		monitor.setIndeterminate(true);
+		monitor.setMessage("Sorting " + sourceInfo.size() + " entries");
 		sourceInfo.sort((i, j) -> Long.compareUnsigned(i.address(), j.address()));
-		monitor.initialize(sourceInfo.size());
+		monitor.setIndeterminate(false);
+		monitor.initialize(sourceInfo.size(), "DWARF: Applying Source Map Info");
+		Map<SourceFileAddr, SourceFile> sfasToSourceFiles = new HashMap<>();
+		Set<SourceFileAddr> badSfas = new HashSet<>();
+		AddressSet warnedAddresses = new AddressSet();
 		for (int i = 0; i < sourceInfo.size(); i++) {
-			monitor.checkCancelled();
 			monitor.increment(1);
 			SourceFileAddr sourceFileAddr = sourceInfo.get(i);
 			if (sourceFileAddr.isEndSequence()) {
 				continue;
 			}
+			if (sourceFileAddr.fileName() == null) {
+				continue;
+			}
+
+			if (badSfas.contains(sourceFileAddr)) {
+				continue;
+			}
+
 			Address addr = dprog.getCodeAddress(sourceFileAddr.address());
+			if (warnedAddresses.contains(addr)) {
+				continue; // only warn once per address
+			}
+
 			if (!currentProgram.getMemory().getExecuteSet().contains(addr)) {
+				if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) {
 					printerr(
 						"entry for non-executable address; skipping: file %s line %d address: %s %x"
 						.formatted(sourceFileAddr.fileName(), sourceFileAddr.lineNum(),
 							addr.toString(), sourceFileAddr.address()));
+				}
+				warnedAddresses.add(addr);
 				continue;
 			}
 
 			long length = getLength(i, sourceInfo);
 			if (length < 0) {
+				if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) {
 					println(
 						"Error computing entry length for file %s line %d address %s %x; replacing" +
 							" with length 0 entry".formatted(sourceFileAddr.fileName(),
-							sourceFileAddr.lineNum(), addr.toString(), sourceFileAddr.address()));
-				length = 0;
+								sourceFileAddr.lineNum(), addr.toString(),
+								sourceFileAddr.address()));
+				}
 			}
 			if (length > ENTRY_MAX_LENGTH) {
+				if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) {
 					println(
 						("entry for file %s line %d address: %s %x length %d too large, replacing " +
 								"with length 0 entry").formatted(sourceFileAddr.fileName(),
 									sourceFileAddr.lineNum(), addr.toString(), sourceFileAddr.address(),
 									length));
-				length = 0;
 				}
-			if (sourceFileAddr.fileName() == null) {
-				continue;
 			}
-			SourceFile source = null;
+
+
+			SourceFile source = sfasToSourceFiles.get(sourceFileAddr);
+			if (source == null) {
+				String path = SourceFileUtils.fixDwarfRelativePath(sourceFileAddr.fileName(),
+					COMPILATION_ROOT_DIRECTORY);
 				try {
 					SourceFileIdType type =
 						sourceFileAddr.md5() == null ? SourceFileIdType.NONE : SourceFileIdType.MD5;
-				source = new SourceFile(sourceFileAddr.fileName(), type, sourceFileAddr.md5());
+					source = new SourceFile(path, type, sourceFileAddr.md5());
 					sourceManager.addSourceFile(source);
+					sfasToSourceFiles.put(sourceFileAddr, source);
 				}
 				catch (IllegalArgumentException e) {
+					if (numErrors++ < MAX_ERROR_MSGS_TO_DISPLAY) {
 						printerr("Exception creating source file %s".formatted(e.getMessage()));
+					}
+					badSfas.add(sourceFileAddr);
 					continue;
 				}
+			}
 			try {
 				sourceManager.addSourceMapEntry(source, sourceFileAddr.lineNum(), addr, length);
 			}
 			catch (IllegalArgumentException e) {
+				if (numErrors++ < MAX_ERROR_MSGS_TO_DISPLAY) {
 					printerr(e.getMessage());
+				}
 				continue;
 			}
 			entryCount++;
 		}
+		if (numWarnings >= MAX_WARNING_MSGS_TO_DISPLAY) {
+			println("Additional warning messages suppressed");
+		}
+		if (numErrors >= MAX_ERROR_MSGS_TO_DISPLAY) {
+			println("Additional error messages suppressed");
+		}
 		println("Added " + entryCount + " source map entries");
+		printf("There were %d errors and %d warnings\n", numErrors,numWarnings);
 	}
 
 	private long getLength(int i, List<SourceFileAddr> allSFA) {

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFImporter.java

@@ -27,13 +27,13 @@ import ghidra.app.util.bin.format.dwarf.line.DWARFLineProgramExecutor;
 import ghidra.app.util.bin.format.golang.GoConstants;
 import ghidra.framework.store.LockException;
 import ghidra.program.database.sourcemap.SourceFile;
-import ghidra.program.model.address.Address;
-import ghidra.program.model.address.AddressOverflowException;
+import ghidra.program.database.sourcemap.SourceFileIdType;
+import ghidra.program.model.address.*;
 import ghidra.program.model.data.*;
-import ghidra.program.model.listing.*;
+import ghidra.program.model.listing.BookmarkType;
+import ghidra.program.model.listing.Program;
 import ghidra.program.model.sourcemap.SourceFileManager;
-import ghidra.util.Msg;
-import ghidra.util.Swing;
+import ghidra.util.*;
 import ghidra.util.exception.*;
 import ghidra.util.task.TaskMonitor;
 import utility.function.Dummy;
@@ -49,6 +49,13 @@ public class DWARFImporter {
 	private DWARFProgram prog;
 	private DWARFDataTypeManager dwarfDTM;
 	private TaskMonitor monitor;
+	private static final int MAX_NUM_SOURCE_LINE_ERROR_REPORTS = 200;
+	private static final int MAX_NUM_SOURCE_LINE_WARNING_REPORTS = 200;
+	private int numSourceLineErrorReports = 0;
+	private int numSourceLineWarningReports = 0;
+
+	// TODO: consider making this an analyzer option
+	public static final String DEFAULT_COMPILATION_DIR = "DWARF_DEFAULT_COMP_DIR";
 
 	public DWARFImporter(DWARFProgram prog, TaskMonitor monitor) {
 		this.prog = prog;
@@ -183,6 +190,17 @@ public class DWARFImporter {
 		return new CategoryPath(newRoot, cpParts.subList(origRootParts.size(), cpParts.size()));
 	}
 
+	/**
+	 * Reads the dwarf source line info and applies it via the program's {@link SourceFileManager}.
+	 * Note that source file paths which are relative after normalization will have all leading
+	 * "." and "/../" entries stripped and then be placed under artificial directories based on
+	 * {@code DEFAULT_COMPILATION_DIR}.
+	 * 
+	 * @param reader reader
+	 * @throws CancelledException if cancelled by user
+	 * @throws IOException if error during reading
+	 * @throws LockException if invoked without exclusive access
+	 */
 	private void addSourceLineInfo(BinaryReader reader)
 			throws CancelledException, IOException, LockException {
 		if (reader == null) {
@@ -191,21 +209,25 @@ public class DWARFImporter {
 		}
 		int entryCount = 0;
 		Program ghidraProgram = prog.getGhidraProgram();
-		BookmarkManager bookmarkManager = ghidraProgram.getBookmarkManager();
 		long maxLength = prog.getImportOptions().getMaxSourceMapEntryLength();
-		boolean errorBookmarks = prog.getImportOptions().isUseBookmarks();
 		List<DWARFCompilationUnit> compUnits = prog.getCompilationUnits();
-		monitor.initialize(compUnits.size(), "Reading DWARF Source Map Info");
+		monitor.initialize(compUnits.size(), "DWARF: Reading Source Map Info");
 		SourceFileManager sourceManager = ghidraProgram.getSourceFileManager();
 		List<SourceFileAddr> sourceInfo = new ArrayList<>();
 		for (DWARFCompilationUnit cu : compUnits) {
 			monitor.increment();
 			sourceInfo.addAll(cu.getLine().getAllSourceFileAddrInfo(cu, reader));
 		}
+		monitor.setIndeterminate(true);
+		monitor.setMessage("Sorting " + sourceInfo.size() + " entries");
 		sourceInfo.sort((i, j) -> Long.compareUnsigned(i.address(), j.address()));
-		monitor.initialize(sourceInfo.size(), "Applying DWARF Source Map Info");
+		monitor.setIndeterminate(false);
+		monitor.initialize(sourceInfo.size(), "DWARF: Applying Source Map Info");
+		Map<SourceFileAddr, SourceFile> sfasToSourceFiles = new HashMap<>();
+		Set<SourceFileAddr> badSfas = new HashSet<>();
+		AddressSet warnedAddresses = new AddressSet();
+
 		for (int i = 0; i < sourceInfo.size() - 1; i++) {
-			monitor.checkCancelled();
 			monitor.increment(1);
 			SourceFileAddr sfa = sourceInfo.get(i);
 			if (SOURCEFILENAMES_IGNORE.contains(sfa.fileName()) ||
@@ -213,43 +235,73 @@ public class DWARFImporter {
 				sfa.isEndSequence()) {
 				continue;
 			}
+			if (sfa.fileName() == null) {
+				continue;
+			}
+			if (badSfas.contains(sfa)) {
+				continue;
+			}
+
 			Address addr = prog.getCodeAddress(sfa.address());
+
+			if (warnedAddresses.contains(addr)) {
+				continue; // only warn once per address
+			}
 			if (!ghidraProgram.getMemory().getExecuteSet().contains(addr)) {
-				String errorString =
+				String warningString =
 					"entry for non-executable address; skipping: file %s line %d address: %s %x"
 							.formatted(sfa.fileName(), sfa.lineNum(), addr.toString(),
 								sfa.address());
 
-				reportError(bookmarkManager, errorString, addr, errorBookmarks);
+				if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) {
+					prog.logWarningAt(addr, addr.toString(), warningString);
+				}
+				warnedAddresses.add(addr);
 				continue;
 			}
 
 			long length = getLength(i, sourceInfo);
 			if (length < 0) {
 				length = 0;
-				String errorString =
+				String warningString =
 					"Error calculating entry length for file %s line %d address %s %x; replacing " +
 						"with length 0 entry".formatted(sfa.fileName(), sfa.lineNum(),
 							addr.toString(), sfa.address());
-				reportError(bookmarkManager, errorString, addr, errorBookmarks);
+				if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) {
+					prog.logWarningAt(addr, addr.toString(), warningString);
+				}
 			}
 			if (length > maxLength) {
-				String errorString = ("entry for file %s line %d address: %s %x length %d too" +
+				String warningString = ("entry for file %s line %d address: %s %x length %d too" +
 					" large, replacing with length 0 entry").formatted(sfa.fileName(),
 						sfa.lineNum(), addr.toString(), sfa.address(), length);
 				length = 0;
-				reportError(bookmarkManager, errorString, addr, errorBookmarks);
+				if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) {
+					prog.logWarningAt(addr, addr.toString(), warningString);
+				}
 			}
 
-			SourceFile source = null;
+			SourceFile source = sfasToSourceFiles.get(sfa);
+			if (source == null) {
+				String path = SourceFileUtils.fixDwarfRelativePath(sfa.fileName(),
+					DEFAULT_COMPILATION_DIR);
 				try {
-				source = new SourceFile(sfa.fileName());
+					SourceFileIdType type =
+						sfa.md5() == null ? SourceFileIdType.NONE : SourceFileIdType.MD5;
+					source = new SourceFile(path, type, sfa.md5());
 					sourceManager.addSourceFile(source);
+					sfasToSourceFiles.put(sfa, source);
 				}
 				catch (IllegalArgumentException e) {
-				Msg.error(this, "Exception creating source file: %s".formatted(e.getMessage()));
+					String errorString = "Exception creating source file: " + e.getMessage();
+					if (numSourceLineErrorReports++ < MAX_NUM_SOURCE_LINE_ERROR_REPORTS) {
+						reportError(errorString, addr);
+					}
+					badSfas.add(sfa);
 					continue;
 				}
+			}
+
 			try {
 				sourceManager.addSourceMapEntry(source, sfa.lineNum(), addr, length);
 			}
@@ -257,18 +309,29 @@ public class DWARFImporter {
 				String errorString = "AddressOverflowException for source map entry %s %d %s %x %d"
 						.formatted(source.getFilename(), sfa.lineNum(), addr.toString(),
 							sfa.address(), length);
-				reportError(bookmarkManager, errorString, addr, errorBookmarks);
+				if (numSourceLineErrorReports++ < MAX_NUM_SOURCE_LINE_ERROR_REPORTS) {
+					reportError(errorString, addr);
+				}
 				continue;
 			}
 			entryCount++;
 		}
+		if (numSourceLineWarningReports >= MAX_NUM_SOURCE_LINE_WARNING_REPORTS) {
+			Msg.warn(this, "Additional warnings suppressed (%d total warnings)"
+					.formatted(numSourceLineWarningReports));
+		}
+		if (numSourceLineErrorReports >= MAX_NUM_SOURCE_LINE_ERROR_REPORTS) {
+			Msg.error(this, "Additional errors suppressed (%d total errors)"
+					.formatted(numSourceLineErrorReports));
+		}
 		Msg.info(this, "Added %d source map entries".formatted(entryCount));
 	}
 
-	private void reportError(BookmarkManager bManager, String errorString, Address addr,
-			boolean errorBookmarks) {
-		if (errorBookmarks) {
-			bManager.setBookmark(addr, BookmarkType.ERROR, DWARFProgram.DWARF_BOOKMARK_CAT,
+	private void reportError(String errorString, Address addr) {
+		if (prog.getImportOptions().isUseBookmarks()) {
+			prog.getGhidraProgram()
+					.getBookmarkManager()
+					.setBookmark(addr, BookmarkType.ERROR, DWARFProgram.DWARF_BOOKMARK_CAT,
 						errorString);
 		}
 		else {

Ghidra/Features/Base/src/main/java/ghidra/util/SourceFileUtils.java

@@ -16,8 +16,11 @@
 package ghidra.util;
 
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.HexFormat;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -33,6 +36,7 @@ import ghidra.program.model.sourcemap.SourceMapEntry;
 public class SourceFileUtils {
 
 	private static HexFormat hexFormat = HexFormat.of();
+	private static Pattern dirNameValidator = Pattern.compile("\\W");
 
 	private SourceFileUtils() {
 		// singleton class
@@ -147,6 +151,58 @@ public class SourceFileUtils {
 		return new SourceLineBounds(min, max);
 	}
 
+	/**
+	 * Corrects potentially relative paths encountered in DWARF debug info.
+	 * Relative paths are based at /{@code baseDir}/.  If normalization of "/../" subpaths
+	 * results in a path "above" /{@code baseDir}/, the returned path will be based at "baseDir_i"
+	 * where i is the count of initial "/../" in the normalized path.
+	 * @param path path to normalize
+	 * @param baseDir name of artificial root directory
+	 * @return normalized path
+	 * @throws IllegalArgumentException if the path is not valid or if baseDir contains a
+	 * non-alphanumeric, non-underscore character
+	 */
+	public static String fixDwarfRelativePath(String path, String baseDir) {
+		if (StringUtils.isEmpty(baseDir)) {
+			throw new IllegalArgumentException("baseDir cannot be empty");
+		}
+		Matcher matcher = dirNameValidator.matcher(baseDir);
+		if (matcher.find()) {
+			throw new IllegalArgumentException(
+				"baseDir must consist of alphanumeric characters or underscores");
+		}
+		boolean based = false;
+		if (path.startsWith("./")) {
+			path = "/" + baseDir + path.substring(1);
+			based = true;
+		}
+		try {
+			URI uri = new URI("file", null, path, null).normalize();
+			path = uri.getPath();
+		}
+		catch (URISyntaxException e) {
+			throw new IllegalArgumentException("path not valid: " + e.getMessage());
+		}
+		int numDotDots = 0;
+		while (path.startsWith("/..")) {
+			path = path.substring(3);
+			numDotDots += 1;
+		}
+		if (numDotDots == 0) {
+			if (!based) {
+				return path; // baseDir not necessary: path normalizes to absolute path without it 
+			}
+			if (path.startsWith("/" + baseDir)) {
+				return path; // adding initial /baseDir was sufficient
+			}
+		}
+		if (based) {
+			numDotDots += 1; //initial baseDir was consumed by interior /../ during normalization
+		}
+		String count = numDotDots == 0 ? "" : "_" + Integer.toString(numDotDots);
+		return "/" + baseDir + count + path;
+	}
+
 	/**
 	 * A record containing the minimum and maximum mapped line numbers
 	 * @param min minimum line number

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/sourcemap/SourceFile.java

@@ -104,6 +104,9 @@ public final class SourceFile implements Comparable<SourceFile> {
 					throw new IllegalArgumentException(
 						"SourceFile URI must represent a file (not a directory)");
 				}
+				if (path.startsWith("/../")) {
+					throw new IllegalArgumentException("path must be absolute after normalization");
+				}
 			}
 			catch (URISyntaxException e) {
 				throw new IllegalArgumentException("path not valid: " + e.getMessage());

Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/program/database/sourcemap/SourceFileTest.java

@@ -52,11 +52,48 @@ public class SourceFileTest extends AbstractSourceFileTest {
 	}
 
 	@Test
-	public void testPathNormalization() {
+	public void testPathNormalizationLinux() {
 		assertEquals("/src/dir1/dir2/file.c",
 			new SourceFile("/src/test/../dir1/test/../dir2/file.c").getPath());
 	}
 
+	@Test(expected = IllegalArgumentException.class)
+	public void testInteriorPathNormalizationLinux() {
+		new SourceFile("/src/../../../file.c");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testInteriorPathNormalizationLinuxUtilsMethod() {
+		SourceFileUtils.getSourceFileFromPathString("/src/../../../file.c");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testInteriorPathNormalizationWindows() {
+		new SourceFile("/c:/src/../../../file.c");
+	}
+
+	@Test
+	public void testFixDwarfRelativePath() {
+		String baseDirName = "root_dir";
+		assertEquals("/src/file.c",
+			SourceFileUtils.fixDwarfRelativePath("/src/file.c", baseDirName));
+		assertEquals("/file.c",
+			SourceFileUtils.fixDwarfRelativePath("/src/../file.c", baseDirName));
+		assertEquals("/root_dir/file.c",
+			SourceFileUtils.fixDwarfRelativePath("./file.c", baseDirName));
+		assertEquals("/root_dir_1/file.c",
+			SourceFileUtils.fixDwarfRelativePath("/../file.c", baseDirName));
+		assertEquals("/root_dir_2/file.c",
+			SourceFileUtils.fixDwarfRelativePath("/.././../file.c", baseDirName));
+		assertEquals("/root_dir_1/file.c",
+			SourceFileUtils.fixDwarfRelativePath("./../file.c", baseDirName));
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testInteriorPathNormalizationWindowsUtilsMethod() {
+		SourceFileUtils.getSourceFileFromPathString("c:\\src\\..\\..\\..\\file.c");
+	}
+
 	@Test
 	public void testGetFilename() {
 		assertEquals("file.c", new SourceFile("/src/test/file.c").getFilename());