diff --git a/Ghidra/Features/Base/ghidra_scripts/DWARFLineInfoSourceMapScript.java b/Ghidra/Features/Base/ghidra_scripts/DWARFLineInfoSourceMapScript.java index 92d3cd901d..5477145c49 100644 --- a/Ghidra/Features/Base/ghidra_scripts/DWARFLineInfoSourceMapScript.java +++ b/Ghidra/Features/Base/ghidra_scripts/DWARFLineInfoSourceMapScript.java @@ -14,12 +14,13 @@ * limitations under the License. */ // Adds DWARF source file line number info to the current program as source map entries. +// A source file that is relative after path normalization will have all leading "." +// and "/../" entries stripped and then be placed under an artificial directory. // Note that you can run this script on a program that has already been analyzed by the // DWARF analyzer. //@category DWARF import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import ghidra.app.script.GhidraScript; import ghidra.app.util.bin.BinaryReader; @@ -30,25 +31,30 @@ import ghidra.app.util.bin.format.dwarf.sectionprovider.DWARFSectionProviderFact import ghidra.framework.store.LockException; import ghidra.program.database.sourcemap.SourceFile; import ghidra.program.database.sourcemap.SourceFileIdType; -import ghidra.program.model.address.Address; -import ghidra.program.model.address.AddressOverflowException; +import ghidra.program.model.address.*; import ghidra.program.model.sourcemap.SourceFileManager; import ghidra.util.Msg; +import ghidra.util.SourceFileUtils; import ghidra.util.exception.CancelledException; public class DWARFLineInfoSourceMapScript extends GhidraScript { public static final int ENTRY_MAX_LENGTH = 1000; + private static final int MAX_ERROR_MSGS_TO_DISPLAY = 25; + private static final int MAX_WARNING_MSGS_TO_DISPLAY = 25; + private static final String COMPILATION_ROOT_DIRECTORY = DWARFImporter.DEFAULT_COMPILATION_DIR; + private int numErrors; + private int numWarnings; @Override protected void run() throws Exception { if (!currentProgram.hasExclusiveAccess()) { Msg.showError(this, null, "Exclusive Access Required", - "Must have exclusive access to a program to add source map info"); + "Must have exclusive access to a program to add source map info"); return; } DWARFSectionProvider dsp = - DWARFSectionProviderFactory.createSectionProviderFor(currentProgram, monitor); + DWARFSectionProviderFactory.createSectionProviderFor(currentProgram, monitor); if (dsp == null) { printerr("Unable to find DWARF information"); return; @@ -69,71 +75,111 @@ public class DWARFLineInfoSourceMapScript extends GhidraScript { return; } int entryCount = 0; - monitor.initialize(reader.length(), "DWARF Source Map Info"); List compUnits = dprog.getCompilationUnits(); SourceFileManager sourceManager = currentProgram.getSourceFileManager(); List sourceInfo = new ArrayList<>(); + monitor.initialize(compUnits.size(), "DWARF: Reading Source Map Info"); for (DWARFCompilationUnit cu : compUnits) { + monitor.increment(); sourceInfo.addAll(cu.getLine().getAllSourceFileAddrInfo(cu, reader)); } + monitor.setIndeterminate(true); + monitor.setMessage("Sorting " + sourceInfo.size() + " entries"); sourceInfo.sort((i, j) -> Long.compareUnsigned(i.address(), j.address())); - monitor.initialize(sourceInfo.size()); + monitor.setIndeterminate(false); + monitor.initialize(sourceInfo.size(), "DWARF: Applying Source Map Info"); + Map sfasToSourceFiles = new HashMap<>(); + Set badSfas = new HashSet<>(); + AddressSet warnedAddresses = new AddressSet(); for (int i = 0; i < sourceInfo.size(); i++) { - monitor.checkCancelled(); monitor.increment(1); SourceFileAddr sourceFileAddr = sourceInfo.get(i); if (sourceFileAddr.isEndSequence()) { continue; } + if (sourceFileAddr.fileName() == null) { + continue; + } + + if (badSfas.contains(sourceFileAddr)) { + continue; + } + Address addr = dprog.getCodeAddress(sourceFileAddr.address()); + if (warnedAddresses.contains(addr)) { + continue; // only warn once per address + } + if (!currentProgram.getMemory().getExecuteSet().contains(addr)) { - printerr( - "entry for non-executable address; skipping: file %s line %d address: %s %x" - .formatted(sourceFileAddr.fileName(), sourceFileAddr.lineNum(), - addr.toString(), sourceFileAddr.address())); + if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) { + printerr( + "entry for non-executable address; skipping: file %s line %d address: %s %x" + .formatted(sourceFileAddr.fileName(), sourceFileAddr.lineNum(), + addr.toString(), sourceFileAddr.address())); + } + warnedAddresses.add(addr); continue; } long length = getLength(i, sourceInfo); if (length < 0) { - println( - "Error computing entry length for file %s line %d address %s %x; replacing" + - " with length 0 entry".formatted(sourceFileAddr.fileName(), - sourceFileAddr.lineNum(), addr.toString(), sourceFileAddr.address())); - length = 0; + if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) { + println( + "Error computing entry length for file %s line %d address %s %x; replacing" + + " with length 0 entry".formatted(sourceFileAddr.fileName(), + sourceFileAddr.lineNum(), addr.toString(), + sourceFileAddr.address())); + } } if (length > ENTRY_MAX_LENGTH) { - println( - ("entry for file %s line %d address: %s %x length %d too large, replacing " + - "with length 0 entry").formatted(sourceFileAddr.fileName(), - sourceFileAddr.lineNum(), addr.toString(), sourceFileAddr.address(), - length)); - length = 0; + if (numWarnings++ < MAX_WARNING_MSGS_TO_DISPLAY) { + println( + ("entry for file %s line %d address: %s %x length %d too large, replacing " + + "with length 0 entry").formatted(sourceFileAddr.fileName(), + sourceFileAddr.lineNum(), addr.toString(), sourceFileAddr.address(), + length)); + } } - if (sourceFileAddr.fileName() == null) { - continue; - } - SourceFile source = null; - try { - SourceFileIdType type = - sourceFileAddr.md5() == null ? SourceFileIdType.NONE : SourceFileIdType.MD5; - source = new SourceFile(sourceFileAddr.fileName(), type, sourceFileAddr.md5()); - sourceManager.addSourceFile(source); - } - catch (IllegalArgumentException e) { - printerr("Exception creating source file %s".formatted(e.getMessage())); - continue; + + + SourceFile source = sfasToSourceFiles.get(sourceFileAddr); + if (source == null) { + String path = SourceFileUtils.fixDwarfRelativePath(sourceFileAddr.fileName(), + COMPILATION_ROOT_DIRECTORY); + try { + SourceFileIdType type = + sourceFileAddr.md5() == null ? SourceFileIdType.NONE : SourceFileIdType.MD5; + source = new SourceFile(path, type, sourceFileAddr.md5()); + sourceManager.addSourceFile(source); + sfasToSourceFiles.put(sourceFileAddr, source); + } + catch (IllegalArgumentException e) { + if (numErrors++ < MAX_ERROR_MSGS_TO_DISPLAY) { + printerr("Exception creating source file %s".formatted(e.getMessage())); + } + badSfas.add(sourceFileAddr); + continue; + } } try { sourceManager.addSourceMapEntry(source, sourceFileAddr.lineNum(), addr, length); } catch (IllegalArgumentException e) { - printerr(e.getMessage()); + if (numErrors++ < MAX_ERROR_MSGS_TO_DISPLAY) { + printerr(e.getMessage()); + } continue; } entryCount++; } + if (numWarnings >= MAX_WARNING_MSGS_TO_DISPLAY) { + println("Additional warning messages suppressed"); + } + if (numErrors >= MAX_ERROR_MSGS_TO_DISPLAY) { + println("Additional error messages suppressed"); + } println("Added " + entryCount + " source map entries"); + printf("There were %d errors and %d warnings\n", numErrors,numWarnings); } private long getLength(int i, List allSFA) { diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFImporter.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFImporter.java index 5e5b4512ce..6372ba65f3 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFImporter.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/dwarf/DWARFImporter.java @@ -27,13 +27,13 @@ import ghidra.app.util.bin.format.dwarf.line.DWARFLineProgramExecutor; import ghidra.app.util.bin.format.golang.GoConstants; import ghidra.framework.store.LockException; import ghidra.program.database.sourcemap.SourceFile; -import ghidra.program.model.address.Address; -import ghidra.program.model.address.AddressOverflowException; +import ghidra.program.database.sourcemap.SourceFileIdType; +import ghidra.program.model.address.*; import ghidra.program.model.data.*; -import ghidra.program.model.listing.*; +import ghidra.program.model.listing.BookmarkType; +import ghidra.program.model.listing.Program; import ghidra.program.model.sourcemap.SourceFileManager; -import ghidra.util.Msg; -import ghidra.util.Swing; +import ghidra.util.*; import ghidra.util.exception.*; import ghidra.util.task.TaskMonitor; import utility.function.Dummy; @@ -49,6 +49,13 @@ public class DWARFImporter { private DWARFProgram prog; private DWARFDataTypeManager dwarfDTM; private TaskMonitor monitor; + private static final int MAX_NUM_SOURCE_LINE_ERROR_REPORTS = 200; + private static final int MAX_NUM_SOURCE_LINE_WARNING_REPORTS = 200; + private int numSourceLineErrorReports = 0; + private int numSourceLineWarningReports = 0; + + // TODO: consider making this an analyzer option + public static final String DEFAULT_COMPILATION_DIR = "DWARF_DEFAULT_COMP_DIR"; public DWARFImporter(DWARFProgram prog, TaskMonitor monitor) { this.prog = prog; @@ -183,6 +190,17 @@ public class DWARFImporter { return new CategoryPath(newRoot, cpParts.subList(origRootParts.size(), cpParts.size())); } + /** + * Reads the dwarf source line info and applies it via the program's {@link SourceFileManager}. + * Note that source file paths which are relative after normalization will have all leading + * "." and "/../" entries stripped and then be placed under artificial directories based on + * {@code DEFAULT_COMPILATION_DIR}. + * + * @param reader reader + * @throws CancelledException if cancelled by user + * @throws IOException if error during reading + * @throws LockException if invoked without exclusive access + */ private void addSourceLineInfo(BinaryReader reader) throws CancelledException, IOException, LockException { if (reader == null) { @@ -191,21 +209,25 @@ public class DWARFImporter { } int entryCount = 0; Program ghidraProgram = prog.getGhidraProgram(); - BookmarkManager bookmarkManager = ghidraProgram.getBookmarkManager(); long maxLength = prog.getImportOptions().getMaxSourceMapEntryLength(); - boolean errorBookmarks = prog.getImportOptions().isUseBookmarks(); List compUnits = prog.getCompilationUnits(); - monitor.initialize(compUnits.size(), "Reading DWARF Source Map Info"); + monitor.initialize(compUnits.size(), "DWARF: Reading Source Map Info"); SourceFileManager sourceManager = ghidraProgram.getSourceFileManager(); List sourceInfo = new ArrayList<>(); for (DWARFCompilationUnit cu : compUnits) { monitor.increment(); sourceInfo.addAll(cu.getLine().getAllSourceFileAddrInfo(cu, reader)); } + monitor.setIndeterminate(true); + monitor.setMessage("Sorting " + sourceInfo.size() + " entries"); sourceInfo.sort((i, j) -> Long.compareUnsigned(i.address(), j.address())); - monitor.initialize(sourceInfo.size(), "Applying DWARF Source Map Info"); + monitor.setIndeterminate(false); + monitor.initialize(sourceInfo.size(), "DWARF: Applying Source Map Info"); + Map sfasToSourceFiles = new HashMap<>(); + Set badSfas = new HashSet<>(); + AddressSet warnedAddresses = new AddressSet(); + for (int i = 0; i < sourceInfo.size() - 1; i++) { - monitor.checkCancelled(); monitor.increment(1); SourceFileAddr sfa = sourceInfo.get(i); if (SOURCEFILENAMES_IGNORE.contains(sfa.fileName()) || @@ -213,43 +235,73 @@ public class DWARFImporter { sfa.isEndSequence()) { continue; } + if (sfa.fileName() == null) { + continue; + } + if (badSfas.contains(sfa)) { + continue; + } + Address addr = prog.getCodeAddress(sfa.address()); + + if (warnedAddresses.contains(addr)) { + continue; // only warn once per address + } if (!ghidraProgram.getMemory().getExecuteSet().contains(addr)) { - String errorString = + String warningString = "entry for non-executable address; skipping: file %s line %d address: %s %x" .formatted(sfa.fileName(), sfa.lineNum(), addr.toString(), sfa.address()); - reportError(bookmarkManager, errorString, addr, errorBookmarks); + if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) { + prog.logWarningAt(addr, addr.toString(), warningString); + } + warnedAddresses.add(addr); continue; } long length = getLength(i, sourceInfo); if (length < 0) { length = 0; - String errorString = + String warningString = "Error calculating entry length for file %s line %d address %s %x; replacing " + "with length 0 entry".formatted(sfa.fileName(), sfa.lineNum(), addr.toString(), sfa.address()); - reportError(bookmarkManager, errorString, addr, errorBookmarks); + if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) { + prog.logWarningAt(addr, addr.toString(), warningString); + } } if (length > maxLength) { - String errorString = ("entry for file %s line %d address: %s %x length %d too" + + String warningString = ("entry for file %s line %d address: %s %x length %d too" + " large, replacing with length 0 entry").formatted(sfa.fileName(), sfa.lineNum(), addr.toString(), sfa.address(), length); length = 0; - reportError(bookmarkManager, errorString, addr, errorBookmarks); + if (numSourceLineWarningReports++ < MAX_NUM_SOURCE_LINE_WARNING_REPORTS) { + prog.logWarningAt(addr, addr.toString(), warningString); + } } - SourceFile source = null; - try { - source = new SourceFile(sfa.fileName()); - sourceManager.addSourceFile(source); - } - catch (IllegalArgumentException e) { - Msg.error(this, "Exception creating source file: %s".formatted(e.getMessage())); - continue; + SourceFile source = sfasToSourceFiles.get(sfa); + if (source == null) { + String path = SourceFileUtils.fixDwarfRelativePath(sfa.fileName(), + DEFAULT_COMPILATION_DIR); + try { + SourceFileIdType type = + sfa.md5() == null ? SourceFileIdType.NONE : SourceFileIdType.MD5; + source = new SourceFile(path, type, sfa.md5()); + sourceManager.addSourceFile(source); + sfasToSourceFiles.put(sfa, source); + } + catch (IllegalArgumentException e) { + String errorString = "Exception creating source file: " + e.getMessage(); + if (numSourceLineErrorReports++ < MAX_NUM_SOURCE_LINE_ERROR_REPORTS) { + reportError(errorString, addr); + } + badSfas.add(sfa); + continue; + } } + try { sourceManager.addSourceMapEntry(source, sfa.lineNum(), addr, length); } @@ -257,19 +309,30 @@ public class DWARFImporter { String errorString = "AddressOverflowException for source map entry %s %d %s %x %d" .formatted(source.getFilename(), sfa.lineNum(), addr.toString(), sfa.address(), length); - reportError(bookmarkManager, errorString, addr, errorBookmarks); + if (numSourceLineErrorReports++ < MAX_NUM_SOURCE_LINE_ERROR_REPORTS) { + reportError(errorString, addr); + } continue; } entryCount++; } + if (numSourceLineWarningReports >= MAX_NUM_SOURCE_LINE_WARNING_REPORTS) { + Msg.warn(this, "Additional warnings suppressed (%d total warnings)" + .formatted(numSourceLineWarningReports)); + } + if (numSourceLineErrorReports >= MAX_NUM_SOURCE_LINE_ERROR_REPORTS) { + Msg.error(this, "Additional errors suppressed (%d total errors)" + .formatted(numSourceLineErrorReports)); + } Msg.info(this, "Added %d source map entries".formatted(entryCount)); } - private void reportError(BookmarkManager bManager, String errorString, Address addr, - boolean errorBookmarks) { - if (errorBookmarks) { - bManager.setBookmark(addr, BookmarkType.ERROR, DWARFProgram.DWARF_BOOKMARK_CAT, - errorString); + private void reportError(String errorString, Address addr) { + if (prog.getImportOptions().isUseBookmarks()) { + prog.getGhidraProgram() + .getBookmarkManager() + .setBookmark(addr, BookmarkType.ERROR, DWARFProgram.DWARF_BOOKMARK_CAT, + errorString); } else { Msg.error(this, errorString); diff --git a/Ghidra/Features/Base/src/main/java/ghidra/util/SourceFileUtils.java b/Ghidra/Features/Base/src/main/java/ghidra/util/SourceFileUtils.java index bff9d2c53b..98cc002509 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/util/SourceFileUtils.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/util/SourceFileUtils.java @@ -16,8 +16,11 @@ package ghidra.util; import java.net.URI; +import java.net.URISyntaxException; import java.util.HexFormat; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; @@ -33,6 +36,7 @@ import ghidra.program.model.sourcemap.SourceMapEntry; public class SourceFileUtils { private static HexFormat hexFormat = HexFormat.of(); + private static Pattern dirNameValidator = Pattern.compile("\\W"); private SourceFileUtils() { // singleton class @@ -147,6 +151,58 @@ public class SourceFileUtils { return new SourceLineBounds(min, max); } + /** + * Corrects potentially relative paths encountered in DWARF debug info. + * Relative paths are based at /{@code baseDir}/. If normalization of "/../" subpaths + * results in a path "above" /{@code baseDir}/, the returned path will be based at "baseDir_i" + * where i is the count of initial "/../" in the normalized path. + * @param path path to normalize + * @param baseDir name of artificial root directory + * @return normalized path + * @throws IllegalArgumentException if the path is not valid or if baseDir contains a + * non-alphanumeric, non-underscore character + */ + public static String fixDwarfRelativePath(String path, String baseDir) { + if (StringUtils.isEmpty(baseDir)) { + throw new IllegalArgumentException("baseDir cannot be empty"); + } + Matcher matcher = dirNameValidator.matcher(baseDir); + if (matcher.find()) { + throw new IllegalArgumentException( + "baseDir must consist of alphanumeric characters or underscores"); + } + boolean based = false; + if (path.startsWith("./")) { + path = "/" + baseDir + path.substring(1); + based = true; + } + try { + URI uri = new URI("file", null, path, null).normalize(); + path = uri.getPath(); + } + catch (URISyntaxException e) { + throw new IllegalArgumentException("path not valid: " + e.getMessage()); + } + int numDotDots = 0; + while (path.startsWith("/..")) { + path = path.substring(3); + numDotDots += 1; + } + if (numDotDots == 0) { + if (!based) { + return path; // baseDir not necessary: path normalizes to absolute path without it + } + if (path.startsWith("/" + baseDir)) { + return path; // adding initial /baseDir was sufficient + } + } + if (based) { + numDotDots += 1; //initial baseDir was consumed by interior /../ during normalization + } + String count = numDotDots == 0 ? "" : "_" + Integer.toString(numDotDots); + return "/" + baseDir + count + path; + } + /** * A record containing the minimum and maximum mapped line numbers * @param min minimum line number diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/sourcemap/SourceFile.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/sourcemap/SourceFile.java index 2a435d800b..77d14571fb 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/sourcemap/SourceFile.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/sourcemap/SourceFile.java @@ -104,6 +104,9 @@ public final class SourceFile implements Comparable { throw new IllegalArgumentException( "SourceFile URI must represent a file (not a directory)"); } + if (path.startsWith("/../")) { + throw new IllegalArgumentException("path must be absolute after normalization"); + } } catch (URISyntaxException e) { throw new IllegalArgumentException("path not valid: " + e.getMessage()); diff --git a/Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/program/database/sourcemap/SourceFileTest.java b/Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/program/database/sourcemap/SourceFileTest.java index 23d3d011fb..4d754485e0 100644 --- a/Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/program/database/sourcemap/SourceFileTest.java +++ b/Ghidra/Test/IntegrationTest/src/test.slow/java/ghidra/program/database/sourcemap/SourceFileTest.java @@ -52,11 +52,48 @@ public class SourceFileTest extends AbstractSourceFileTest { } @Test - public void testPathNormalization() { + public void testPathNormalizationLinux() { assertEquals("/src/dir1/dir2/file.c", new SourceFile("/src/test/../dir1/test/../dir2/file.c").getPath()); } + @Test(expected = IllegalArgumentException.class) + public void testInteriorPathNormalizationLinux() { + new SourceFile("/src/../../../file.c"); + } + + @Test(expected = IllegalArgumentException.class) + public void testInteriorPathNormalizationLinuxUtilsMethod() { + SourceFileUtils.getSourceFileFromPathString("/src/../../../file.c"); + } + + @Test(expected = IllegalArgumentException.class) + public void testInteriorPathNormalizationWindows() { + new SourceFile("/c:/src/../../../file.c"); + } + + @Test + public void testFixDwarfRelativePath() { + String baseDirName = "root_dir"; + assertEquals("/src/file.c", + SourceFileUtils.fixDwarfRelativePath("/src/file.c", baseDirName)); + assertEquals("/file.c", + SourceFileUtils.fixDwarfRelativePath("/src/../file.c", baseDirName)); + assertEquals("/root_dir/file.c", + SourceFileUtils.fixDwarfRelativePath("./file.c", baseDirName)); + assertEquals("/root_dir_1/file.c", + SourceFileUtils.fixDwarfRelativePath("/../file.c", baseDirName)); + assertEquals("/root_dir_2/file.c", + SourceFileUtils.fixDwarfRelativePath("/.././../file.c", baseDirName)); + assertEquals("/root_dir_1/file.c", + SourceFileUtils.fixDwarfRelativePath("./../file.c", baseDirName)); + } + + @Test(expected = IllegalArgumentException.class) + public void testInteriorPathNormalizationWindowsUtilsMethod() { + SourceFileUtils.getSourceFileFromPathString("c:\\src\\..\\..\\..\\file.c"); + } + @Test public void testGetFilename() { assertEquals("file.c", new SourceFile("/src/test/file.c").getFilename());