Merge remote-tracking branch 'origin/GP-4145_dev747368_external_symbol_resolver--SQUASHED'

This commit is contained in:
Ryan Kurtz 2024-02-08 13:19:01 -05:00
commit 9c31aa48d8
3 changed files with 427 additions and 235 deletions

View file

@ -15,21 +15,16 @@
*/ */
package ghidra.app.plugin.core.analysis; package ghidra.app.plugin.core.analysis;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import ghidra.app.services.*; import ghidra.app.services.*;
import ghidra.app.util.importer.MessageLog; import ghidra.app.util.importer.MessageLog;
import ghidra.app.util.opinion.*; import ghidra.app.util.opinion.ElfLoader;
import ghidra.framework.model.*; import ghidra.app.util.opinion.MachoLoader;
import ghidra.framework.options.Options; import ghidra.framework.options.Options;
import ghidra.program.model.address.AddressSetView; import ghidra.program.model.address.AddressSetView;
import ghidra.program.model.listing.Library;
import ghidra.program.model.listing.Program; import ghidra.program.model.listing.Program;
import ghidra.program.util.ExternalSymbolResolver; import ghidra.program.util.ExternalSymbolResolver;
import ghidra.util.Msg;
import ghidra.util.exception.CancelledException; import ghidra.util.exception.CancelledException;
import ghidra.util.exception.VersionException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
/** /**
@ -71,65 +66,16 @@ public class ExternalSymbolResolverAnalyzer extends AbstractAnalyzer {
public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log) public boolean added(Program program, AddressSetView set, TaskMonitor monitor, MessageLog log)
throws CancelledException { throws CancelledException {
Object consumer = new Object(); try (ExternalSymbolResolver esr = new ExternalSymbolResolver(
log = new MessageLog(); // For now, we don't want the analysis log spammed program.getDomainFile().getParent().getProjectData(), monitor)) {
ProjectData projectData = program.getDomainFile().getParent().getProjectData(); esr.addProgramToFixup(program);
List<Loaded<Program>> loadedPrograms = new ArrayList<>(); esr.fixUnresolvedExternalSymbols();
esr.logInfo(s -> Msg.info(this, s), false);
// Add program to list if (esr.hasProblemLibraries()) {
loadedPrograms.add(new Loaded<>(program, program.getName(), // causes a popup message at end of analysis session
program.getDomainFile().getParent().getPathname())); esr.logInfo(log::appendMsg, true);
// Add external libraries to list
for (Library extLibrary : ExternalSymbolResolver.getLibrarySearchList(program)) {
monitor.checkCancelled();
String libPath = extLibrary.getAssociatedProgramPath();
if (libPath == null) {
continue;
} }
DomainFile libDomainFile = projectData.getFile(libPath);
if (libDomainFile == null) {
log.appendMsg("Referenced external program not found: " + libPath);
continue;
}
try {
DomainObject libDomainObject =
libDomainFile.getDomainObject(consumer, false, false, monitor);
if (libDomainObject instanceof Program p) {
loadedPrograms.add(new Loaded<>(p, libDomainFile.getName(),
libDomainFile.getParent().getPathname()));
}
else {
libDomainObject.release(consumer);
log.appendMsg("Referenced external program is not a program: " + libPath);
}
}
catch (IOException e) {
log.appendMsg("Failed to open library dependency project file: " +
libDomainFile.getPathname());
}
catch (VersionException e) {
log.appendMsg(
"Referenced external program requires updgrade, unable to consider symbols: " +
libPath);
}
}
// Resolve symbols
try {
ExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, false, log,
monitor);
return true; return true;
} }
catch (IOException e) {
return false;
}
finally {
for (int i = 1; i < loadedPrograms.size(); i++) {
loadedPrograms.get(i).release(consumer);
}
}
} }
} }

View file

@ -158,8 +158,16 @@ public class ElfLoader extends AbstractLibrarySupportLoader {
throws CancelledException, IOException { throws CancelledException, IOException {
super.postLoadProgramFixups(loadedPrograms, project, options, messageLog, monitor); super.postLoadProgramFixups(loadedPrograms, project, options, messageLog, monitor);
ExternalSymbolResolver.fixUnresolvedExternalSymbols(loadedPrograms, true, messageLog, try (ExternalSymbolResolver esr =
monitor); new ExternalSymbolResolver(project.getProjectData(), monitor)) {
for (Loaded<Program> loadedProgram : loadedPrograms) {
esr.addProgramToFixup(
loadedProgram.getProjectFolderPath() + loadedProgram.getName(),
loadedProgram.getDomainObject());
}
esr.fixUnresolvedExternalSymbols();
esr.logInfo(messageLog::appendMsg, true);
}
} }
@Override @Override

View file

@ -15,13 +15,13 @@
*/ */
package ghidra.program.util; package ghidra.program.util;
import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.function.Consumer;
import db.Transaction; import db.Transaction;
import ghidra.app.util.importer.MessageLog; import ghidra.framework.model.*;
import ghidra.app.util.opinion.Loaded;
import ghidra.framework.options.Options; import ghidra.framework.options.Options;
import ghidra.program.model.listing.*; import ghidra.program.model.listing.*;
import ghidra.program.model.symbol.*; import ghidra.program.model.symbol.*;
@ -30,8 +30,15 @@ import ghidra.util.StringUtilities;
import ghidra.util.exception.*; import ghidra.util.exception.*;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
public class ExternalSymbolResolver { /**
* Moves dangling external function symbols found in the {@link Library#UNKNOWN EXTERNAL/UNKNOWN}
* namespace into the namespace of the external library that publishes a matching symbol.
* <p>
* This uses an ordered list of external library names that was attached to the program during
* import by the Elf or Macho loader (see {@link #REQUIRED_LIBRARY_PROPERTY_PREFIX}).
*
*/
public class ExternalSymbolResolver implements Closeable {
private final static String REQUIRED_LIBRARY_PROPERTY_PREFIX = "Required Library ["; private final static String REQUIRED_LIBRARY_PROPERTY_PREFIX = "Required Library [";
/** /**
@ -45,89 +52,317 @@ public class ExternalSymbolResolver {
StringUtilities.pad("" + libraryIndex, ' ', 4)); StringUtilities.pad("" + libraryIndex, ' ', 4));
} }
private final ProjectData projectData;
private final TaskMonitor monitor;
private final List<ProgramSymbolResolver> programsToFix = new ArrayList<>();
private final Map<String, Program> loadedPrograms = new HashMap<>();
private final Map<String, Throwable> problemLibraries = new HashMap<>();
public ExternalSymbolResolver(ProjectData projectData, TaskMonitor monitor) {
this.projectData = projectData;
this.monitor = monitor;
}
/** /**
* Links unresolved symbols to the first symbol found in the (ordered) linked * Queues a program into this session that will be fixed when {@link #fixUnresolvedExternalSymbols()}
* libraries (saved in the program's properties as {@value #REQUIRED_LIBRARY_PROPERTY_PREFIX}). * is called.
* <p> * <p>
* The ordering and precedence logic is loader specific though no particular binary formats * The program should be fully persisted to the project if using this method, otherwise use
* are parsed or required. * {@link #addProgramToFixup(String, Program)} and specify the pathname the program will
* <p> * be saved to.
* The program's external libraries need to already be populated with paths to
* already existing / imported libraries.
* *
* @param loadedPrograms The {@link Loaded} {@link Program}s to fix. The first entry is the * @param program {@link Program} to fix
* "primary" {@link Loaded} {@link Program}.
* @param fixAll True if all of the {@link Loaded} {@link Program}s should be fixed;
* false if just the "primary" {@link Loaded} {@link Program} should be fixed.
* @param messageLog {@link MessageLog} to write info message to.
* @param monitor {@link TaskMonitor} to watch for cancel and update with progress.
* @throws CancelledException if user cancels
* @throws IOException if error reading
*/ */
public static void fixUnresolvedExternalSymbols(List<Loaded<Program>> loadedPrograms, public void addProgramToFixup(Program program) {
boolean fixAll, MessageLog messageLog, TaskMonitor monitor) addProgramToFixup(program.getDomainFile().getPathname(), program);
throws CancelledException, IOException {
Map<String, Loaded<Program>> loadedByPath = loadedPrograms.stream()
.collect(Collectors.toMap(
loaded -> loaded.getProjectFolderPath() + loaded.getName(), loaded -> loaded));
List<Loaded<Program>> fixupList =
loadedPrograms.subList(0, fixAll ? loadedPrograms.size() : 1);
monitor.initialize(fixupList.size());
for (Loaded<Program> loadedProgram : fixupList) {
Program program = loadedProgram.getDomainObject();
Collection<Long> unresolvedExternalFunctionIds =
getUnresolvedExternalFunctionIds(program);
if (unresolvedExternalFunctionIds.size() == 0) {
continue;
} }
List<Library> libSearchList = getLibrarySearchList(program); /**
if (libSearchList.isEmpty()) { * Queues a program into this session that will be fixed when {@link #fixUnresolvedExternalSymbols()}
continue; * is called.
*
* @param programPath string project path to the program
* @param program {@link Program} to fix
*/
public void addProgramToFixup(String programPath, Program program) {
programsToFix.add(new ProgramSymbolResolver(program, programPath));
addLoadedProgram(programPath, program);
} }
/**
* Adds an already opened program to this session, allowing it to be used as an external
* library without needing to look it up in the current project.
*
* @param programPath project path to already opened program
* @param program {@link Program}
*/
public void addLoadedProgram(String programPath, Program program) {
if (loadedPrograms.put(programPath, program) == null) {
program.addConsumer(this);
}
}
/**
* Returns true if there was an error encountered when trying to open an external library.
*
* @return boolean flag, true if there was a problem opening an external library
*/
public boolean hasProblemLibraries() {
return !problemLibraries.isEmpty();
}
@Override
public void close() {
for (Program prog : loadedPrograms.values()) {
prog.release(this);
}
programsToFix.clear();
loadedPrograms.clear();
}
/**
* Resolves any unresolved external symbols in each program that has been queued up via
* {@link #addProgramToFixup(String, Program)}.
*
* @throws CancelledException if cancelled
*/
public void fixUnresolvedExternalSymbols() throws CancelledException {
for (ProgramSymbolResolver psr : programsToFix) {
psr.resolveExternalSymbols();
}
}
/**
* Logs information about the libraries and symbols that were found during the fixup.
*
* @param logger consumer that will log a string
* @param shortSummary boolean flag, if true individual symbol names will be omitted
*/
public void logInfo(Consumer<String> logger, boolean shortSummary) {
for (ProgramSymbolResolver psr : programsToFix) {
psr.log(logger, shortSummary);
}
}
/**
* Fetches a program from a cache of Program instances. If the requested program
* isn't currently in the cache, it will be opened (if possible).
* <p>
* This cache of programs are pinned by registering a consumer on the program, and will be
* released during {@link #close()} of this ExternalSymbolServer instance.
* <p>
* This cache is shared between all ProgramSymbolResolver instances (that were created
* by calling {@link #addProgramToFixup(String, Program)}).
*
* @param libPath project path to a library program
* @return {@link Program}, or null if not found or other error during opening
* @throws CancelledException if cancelled
*/
protected Program getLibraryProgram(String libPath) throws CancelledException {
Program result = loadedPrograms.get(libPath);
if (result == null && !problemLibraries.containsKey(libPath)) {
result = openLibraryFile(projectData.getFile(libPath), libPath);
if (result != null) {
loadedPrograms.put(libPath, result);
}
}
return result;
}
/**
* Opens a library binary.
*
* @param libDf optional, reference to a the DomainFile that was found in a project. If null
* (meaning a lookup in the project failed to find a matching file), libPath will be used when
* creating error strings that reference the problematic file
* @param libPath project path for the DomainFile
* @return a opened {@link Program}
* @throws CancelledException if cancelled
*/
protected Program openLibraryFile(DomainFile libDf, String libPath) throws CancelledException {
try {
if (libDf == null) {
throw new IOException("Dangling external path: " + libPath);
}
DomainObject libDo = libDf.getDomainObject(this, false, false, monitor);
if (libDo instanceof Program p) {
return p;
}
libDo.release(this);
throw new IOException("Referenced external program is not a program: " + libPath);
}
catch (IOException | VersionException e) {
problemLibraries.put(libPath, e);
}
return null;
}
/**
* Represents a program that needs its external symbols to be fixed.
*/
private class ProgramSymbolResolver {
record ExtLibInfo(String name, Library lib, String programPath, Program program,
List<String> resolvedSymbols, Throwable problem) {
String getProblemMessage() {
if (problem instanceof VersionException ve) {
return getVersionError(ve);
}
return problem != null ? problem.getMessage() : "";
}
String getLibPath() {
return programPath != null ? programPath : "missing";
}
String getVersionError(VersionException ve) {
String versionType = switch (ve.getVersionIndicator()) {
case VersionException.NEWER_VERSION -> " newer";
case VersionException.OLDER_VERSION -> "n older";
default -> "n unknown";
};
String upgradeMsg = ve.isUpgradable() ? " (upgrade is possible)" : "";
return "skipped: file was created with a%s version of Ghidra%s"
.formatted(versionType, upgradeMsg);
}
}
Program program;
String programPath;
int externalSymbolCount;
List<Long> unresolvedExternalFunctionIds;
List<ExtLibInfo> extLibs = new ArrayList<>();
private ProgramSymbolResolver(Program program, String programPath) {
this.program = program;
this.programPath = programPath;
}
private int getResolvedSymbolCount() {
return externalSymbolCount - unresolvedExternalFunctionIds.size();
}
private void log(Consumer<String> logger, boolean shortSummary) {
boolean changed = unresolvedExternalFunctionIds.size() != externalSymbolCount;
if (extLibs.isEmpty() && externalSymbolCount == 0) {
return;
}
else if (!changed && !hasSomeLibrariesConfigured()) {
logger.accept(
"Resolving External Symbols of [%s] - %d unresolved symbols, no external libraries configured - skipping"
.formatted(programPath, externalSymbolCount));
return;
}
logger.accept("Resolving External Symbols of [%s]%s".formatted(programPath,
shortSummary ? " - Summary" : ""));
logger.accept("\t%d external symbols resolved, %d remain unresolved"
.formatted(getResolvedSymbolCount(), unresolvedExternalFunctionIds.size()));
for (ExtLibInfo extLib : extLibs) {
if (extLib.problem != null) {
logger.accept("\t[%s] -> %s, %s".formatted(extLib.name, extLib.getLibPath(),
extLib.getProblemMessage()));
}
else if (extLib.programPath != null) {
logger.accept("\t[%s] -> %s, %d new symbols resolved".formatted(extLib.name,
extLib.getLibPath(), extLib.resolvedSymbols.size()));
}
else {
logger.accept("\t[%s] -> %s".formatted(extLib.name, extLib.getLibPath()));
}
if (!shortSummary) {
for (String symbolName : extLib.resolvedSymbols) {
logger.accept("\t\t[%s]".formatted(symbolName));
}
}
}
if (!shortSummary && changed) {
if (!unresolvedExternalFunctionIds.isEmpty()) {
logger.accept("\tUnresolved remaining %d:"
.formatted(unresolvedExternalFunctionIds.size()));
SymbolTable symbolTable = program.getSymbolTable();
for (Long symId : unresolvedExternalFunctionIds) {
Symbol s = symbolTable.getSymbol(symId);
logger.accept("\t\t[%s]".formatted(s.getName()));
}
}
}
}
private boolean hasSomeLibrariesConfigured() {
for (ExtLibInfo extLib : extLibs) {
if (extLib.program != null || extLib.problem != null ||
extLib.programPath != null) {
return true;
}
}
return false;
}
private void resolveExternalSymbols() throws CancelledException {
unresolvedExternalFunctionIds = getUnresolvedExternalFunctionIds();
externalSymbolCount = unresolvedExternalFunctionIds.size();
if (unresolvedExternalFunctionIds.isEmpty()) {
return;
}
extLibs = getLibsToSearch();
if (!extLibs.isEmpty()) {
try (Transaction tx = program.openTransaction("Resolve External Symbols")) { try (Transaction tx = program.openTransaction("Resolve External Symbols")) {
for (ExtLibInfo extLib : extLibs) {
messageLog.appendMsg("----- [" + program.getName() + "] Resolve " +
unresolvedExternalFunctionIds.size() + " external symbols -----");
for (Library extLibrary : libSearchList) {
monitor.checkCancelled(); monitor.checkCancelled();
String libName = extLibrary.getName(); resolveSymbolsToLibrary(extLib);
String libPath = extLibrary.getAssociatedProgramPath();
if (libPath == null) {
continue;
}
Loaded<Program> loadedLib = loadedByPath.get(libPath);
if (loadedLib == null) {
messageLog.appendMsg("Referenced external program not found: " + libName);
continue;
}
Program libProgram = loadedLib.getDomainObject();
monitor.setMessage("Resolving symbols published by library " + libName);
resolveSymbolsToLibrary(program, unresolvedExternalFunctionIds, extLibrary,
libProgram, messageLog, monitor);
}
messageLog.appendMsg("Unresolved external symbols which remain: " +
unresolvedExternalFunctionIds.size());
} }
} }
} }
private static void resolveSymbolsToLibrary(Program program, }
Collection<Long> unresolvedExternalFunctionIds, Library extLibrary, Program libProgram,
MessageLog messageLog, TaskMonitor monitor) throws CancelledException { /**
int libResolvedCount = 0; * Returns an ordered list of external libraries that need to be searched.
*
* @return list of ExtLibInfo elements, each representing an external library dependency
* found in the {@link #program}
* @throws CancelledException if cancelled
*/
private List<ExtLibInfo> getLibsToSearch() throws CancelledException {
List<ExtLibInfo> result = new ArrayList<>();
ExternalManager externalManager = program.getExternalManager();
for (String libName : getOrderedRequiredLibraryNames()) {
Library lib = externalManager.getExternalLibrary(libName);
String libPath = lib != null ? lib.getAssociatedProgramPath() : null;
Program libProg = libPath != null ? getLibraryProgram(libPath) : null;
Throwable problem =
libProg == null && libPath != null ? problemLibraries.get(libPath) : null;
result.add(
new ExtLibInfo(libName, lib, libPath, libProg, new ArrayList<>(), problem));
}
return result;
}
/**
* Moves unresolved functions from the EXTERNAL/UNKNOWN namespace to the namespace of the
* external library if the extLib publishes a symbol with a matching name.
*
* @param extLib {@link ExtLibInfo} representing an external library
* @throws CancelledException if cancelled
*/
private void resolveSymbolsToLibrary(ExtLibInfo extLib) throws CancelledException {
if (extLib.program == null) {
// can't do anything if the external library doesn't have a valid program associated
return;
}
ExternalManager externalManager = program.getExternalManager(); ExternalManager externalManager = program.getExternalManager();
SymbolTable symbolTable = program.getSymbolTable(); SymbolTable symbolTable = program.getSymbolTable();
Iterator<Long> idIterator = unresolvedExternalFunctionIds.iterator(); for (Iterator<Long> idIterator = unresolvedExternalFunctionIds.iterator(); idIterator
while (idIterator.hasNext()) { .hasNext();) {
monitor.checkCancelled(); monitor.checkCancelled();
Symbol s = symbolTable.getSymbol(idIterator.next()); Symbol s = symbolTable.getSymbol(idIterator.next());
if (s == null || !s.isExternal() || s.getSymbolType() != SymbolType.FUNCTION) { if (s == null || !s.isExternal() || s.getSymbolType() != SymbolType.FUNCTION) {
@ -138,48 +373,36 @@ public class ExternalSymbolResolver {
} }
ExternalLocation extLoc = externalManager.getExternalLocation(s); ExternalLocation extLoc = externalManager.getExternalLocation(s);
if (s.getSource() == SourceType.DEFAULT || String extLocName =
!isLocationContainedInLibrary(libProgram, extLoc)) { Objects.requireNonNullElse(extLoc.getOriginalImportedName(), extLoc.getLabel());
continue; if (isExportedSymbol(extLib.program, extLocName)) {
}
try { try {
s.setNamespace(extLibrary); s.setNamespace(extLib.lib);
idIterator.remove(); idIterator.remove();
libResolvedCount++; extLib.resolvedSymbols.add(s.getName());
Msg.debug(ExternalSymbolResolver.class, "External symbol " + extLoc.getLabel() +
" resolved to " + extLibrary.getName());
} }
catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) { catch (DuplicateNameException | InvalidInputException
| CircularDependencyException e) {
Msg.error(ExternalSymbolResolver.class, Msg.error(ExternalSymbolResolver.class,
"Error setting external symbol namespace for " + extLoc.getLabel(), e); "Error setting external symbol namespace for " + extLoc.getLabel(), e);
} }
} }
messageLog.appendMsg( }
"Resolved " + libResolvedCount + " symbols to library " + extLibrary.getName());
} }
private static boolean isLocationContainedInLibrary(Program libProgram, /**
ExternalLocation extLoc) { * Returns a list of all external functions under the EXTERNAL/UNKNOWN library.
*
String name = extLoc.getOriginalImportedName(); * @return list of func ids that need to be fixed
if (name == null) { */
name = extLoc.getLabel(); private List<Long> getUnresolvedExternalFunctionIds() {
}
for (Symbol s : libProgram.getSymbolTable().getLabelOrFunctionSymbols(name, null)) {
if (s.isExternalEntryPoint()) {
return true;
}
}
return false;
}
private static Collection<Long> getUnresolvedExternalFunctionIds(Program program) {
List<Long> symbolIds = new ArrayList<>(); List<Long> symbolIds = new ArrayList<>();
ExternalManager externalManager = program.getExternalManager(); ExternalManager externalManager = program.getExternalManager();
Library library = externalManager.getExternalLibrary(Library.UNKNOWN); Library library = externalManager.getExternalLibrary(Library.UNKNOWN);
if (library != null) { if (library != null) {
for (Symbol s : program.getSymbolTable().getSymbols(library)) { for (Symbol s : program.getSymbolTable().getSymbols(library)) {
if (s.getSymbolType() == SymbolType.FUNCTION) { if (s.getSymbolType() == SymbolType.FUNCTION &&
s.getSource() != SourceType.DEFAULT) {
symbolIds.add(s.getID()); symbolIds.add(s.getID());
} }
} }
@ -187,7 +410,14 @@ public class ExternalSymbolResolver {
return symbolIds; return symbolIds;
} }
private static Collection<String> getOrderedLibraryNamesNeeded(Program program) { /**
* Returns an ordered list of library names, as specified by the logic/rules of the original
* operating system's loader (eg. Elf / MachO dynamic library loading / symbol resolving
* rules)
*
* @return list of library names, in original order
*/
private Collection<String> getOrderedRequiredLibraryNames() {
TreeMap<Integer, String> orderLibraryMap = new TreeMap<>(); TreeMap<Integer, String> orderLibraryMap = new TreeMap<>();
Options options = program.getOptions(Program.PROGRAM_INFO); Options options = program.getOptions(Program.PROGRAM_INFO);
for (String optionName : options.getOptionNames()) { for (String optionName : options.getOptionNames()) {
@ -219,15 +449,23 @@ public class ExternalSymbolResolver {
return orderLibraryMap.values(); return orderLibraryMap.values();
} }
public static List<Library> getLibrarySearchList(Program program) { }
List<Library> result = new ArrayList<>();
ExternalManager externalManager = program.getExternalManager(); /**
for (String libName : getOrderedLibraryNamesNeeded(program)) { * Returns true if the specified program publishes a symbol with the specified name.
Library lib = externalManager.getExternalLibrary(libName); *
if (lib != null) { * @param program {@link Program}
result.add(lib); * @param name symbol name
* @return true if program publishes a symbol the specified name
*/
private static boolean isExportedSymbol(Program program, String name) {
for (Symbol s : program.getSymbolTable().getLabelOrFunctionSymbols(name, null)) {
if (s.isExternalEntryPoint()) {
return true;
} }
} }
return result; return false;
} }
} }