GT-3462 Corrected missing 0-length classes and structures from PDB XML

output.
Corrected unresolved dependency issue caused by inconsistent datatype
name mutation, etc.

Conflicts:
	Ghidra/Features/PDB/src/main/java/ghidra/app/util/bin/format/pdb/PdbParser.java
This commit is contained in:
ghidra1 2020-01-16 19:11:46 -05:00
parent 6c6b0371c4
commit 3740146b33
15 changed files with 142 additions and 88 deletions

View file

@ -151,7 +151,7 @@ public class PdbAnalyzer extends AbstractAnalyzer {
return true;
}
catch (PdbException e) {
message = "PDB issue: " + e.getMessage();
message = e.getMessage();
log.appendMsg(getName(), message);
log.setStatus(message);
return false;

View file

@ -64,7 +64,7 @@ public class ApplyDataTypes {
compositeQueue.clear();
}
private List<CompositeDefinition> getCompositeDefinitionsInpostDependencyOrder(
private List<CompositeDefinition> getCompositeDefinitionsInPostDependencyOrder(
TaskMonitor monitor) {
JungDirectedGraph<CompositeDefinition, GEdge<CompositeDefinition>> graph =
@ -102,7 +102,7 @@ public class ApplyDataTypes {
monitor.setMessage("Order PDB datatypes... ");
List<CompositeDefinition> verticesInPostOrder =
getCompositeDefinitionsInpostDependencyOrder(monitor);
getCompositeDefinitionsInPostDependencyOrder(monitor);
monitor.setMessage("Building PDB datatypes... ");
@ -116,7 +116,7 @@ public class ApplyDataTypes {
!cachedDataType.getCategoryPath().equals(
pdbParser.getCategory(symbolPath.getParent(), true)) ||
!pdbParser.isCorrectKind(cachedDataType, compositeDefinition.kind)) {
log.appendMsg("Error: Conflicting data type name: " + compositeDefinition.name);
log.appendMsg("PDB", "Conflicting data type name: " + compositeDefinition.name);
continue;
}
Composite composite = (Composite) cachedDataType;
@ -169,7 +169,7 @@ public class ApplyDataTypes {
compositeQueue.put(compositeDefinition.name, compositeDefinition);
if (pdbParser.getCachedDataType(compositeDefinition.name) != null) {
log.appendMsg("Error: Data type name collision - unable to define " +
log.appendMsg("PDB", "Data type name collision - unable to define " +
compositeDefinition.kind.getCamelName() + ": " + compositeDefinition.name);
continue;
}
@ -181,8 +181,8 @@ public class ApplyDataTypes {
Composite composite =
pdbParser.createComposite(compositeDefinition.kind, compositeDefinition.name);
if (composite == null) {
log.appendMsg("Unsupported datatype kind (" + compositeDefinition.kind + "): " +
compositeDefinition.name);
log.appendMsg("PDB", "Unsupported datatype kind (" + compositeDefinition.kind +
"): " + compositeDefinition.name);
continue;
}
// if (!isClasses) {

View file

@ -77,7 +77,7 @@ class ApplyEnums {
enumdt.add(name, memberValue);
}
catch (Exception e) {
log.appendMsg(e.getMessage());
log.appendMsg("PDB", "Enum " + enumdt.getName() + ": " + e.getMessage());
}
}
}

View file

@ -62,9 +62,9 @@ class ApplyLineNumbers {
// "REP" (f3) portion (beginning) of the instruction.
CodeUnit cu = program.getListing().getCodeUnitContaining(address);
if (cu == null) {
log.appendMsg(
"Could not apply source code line number found in PDB (no code unit found at " +
address + ")");
log.appendMsg("PDB",
"Could not apply source code line number (no code unit found at " + address +
")");
}
else {
cu.setProperty("Source Path", sourcefileName);

View file

@ -41,9 +41,21 @@ class ApplyStackVariables {
this.function = function;
}
private boolean isParameterRecoverySupported() {
// NOTE: this is a temporary solution to the lack of proper register
// variable support
Program p = function.getProgram();
if (p.getDefaultPointerSize() != 32) {
return false;
}
return "x86".equals(p.getLanguage().getProcessor().toString());
}
void applyTo(TaskMonitor monitor, MessageLog log) throws CancelledException {
int frameBase = getFrameBaseOffset(monitor);
boolean skipParameters = !isParameterRecoverySupported();
while (xmlParser.hasNext()) {
monitor.checkCanceled();
@ -70,13 +82,26 @@ class ApplyStackVariables {
}
if (PdbKind.OBJECT_POINTER == member.kind) {
if (skipParameters) {
xmlParser.next();
continue;
}
createRegisterParameter(member.memberName, dt, log);
}
else if (PdbKind.PARAMETER == member.kind) {
if (skipParameters) {
xmlParser.next();
continue;
}
createStackVariable(member.memberName, frameBase + member.memberOffset, dt, log);
}
else if (PdbKind.LOCAL == member.kind) {
createStackVariable(member.memberName, frameBase + member.memberOffset, dt, log);
int stackOffset = frameBase + member.memberOffset;
if (skipParameters && function.getStackFrame().isParameterOffset(stackOffset)) {
xmlParser.next();
continue;
}
createStackVariable(member.memberName, stackOffset, dt, log);
}
xmlParser.next();//stack variable number end tag
@ -149,7 +174,7 @@ class ApplyStackVariables {
}
}
catch (Exception e) {
log.appendMsg(
log.appendMsg("PDB",
"Unable to create register variable " + name + " in " + function.getName());
}
return null;
@ -183,8 +208,8 @@ class ApplyStackVariables {
}
}
catch (Exception e) {
log.appendMsg("Unable to create stack variable " + name + " at offset " + offset +
" in " + function.getName());
log.appendMsg("PDB", "Unable to create stack variable " + name + " at offset " +
offset + " in " + function.getName());
}
return variable;
}
@ -192,12 +217,12 @@ class ApplyStackVariables {
private DataType getDataType(PdbXmlMember member, MessageLog log) throws CancelledException {
WrappedDataType wrappedDataType = pdbParser.findDataType(member.memberDataTypeName);
if (wrappedDataType == null) {
log.appendMsg("Error: failed to resolve data type for " + member.kind + ": " +
log.appendMsg("PDB", "Failed to resolve data type for " + member.kind + ": " +
member.memberDataTypeName);
return null;
}
if (wrappedDataType.isZeroLengthArray()) {
log.appendMsg("Error: zero length array not supported for for " + member.kind + ": " +
log.appendMsg("PDB", "Zero length array not supported for for " + member.kind + ": " +
member.memberDataTypeName);
return null;
}

View file

@ -70,7 +70,8 @@ class ApplySymbols {
int length = XmlUtilities.parseInt(elem.getAttribute("length"));
String tag = elem.getAttribute("tag");
// String kind = elem.getAttribute("kind");
String datatype = elem.getAttribute("datatype");
String datatype =
SymbolUtilities.replaceInvalidChars(elem.getAttribute("datatype"), false);
// String undecorated = elem.getAttribute("undecorated");
tagSet.add(tag);
@ -116,9 +117,8 @@ class ApplySymbols {
}
// Don't create label for Data since a separate symbol should also exist with a better name
if (!"Data".equals(tag) &&
!pdbParser.createSymbol(address, name, forcePrimary, log)) {
log.appendMsg("Unable to create symbol " + name + " at " + address);
if (!"Data".equals(tag)) {
pdbParser.createSymbol(address, name, forcePrimary, log);
}
////////////

View file

@ -48,8 +48,8 @@ class ApplyTypeDefs {
* @throws CancelledException if monitor is cancelled
* @throws SAXParseException PDB XML parse failure
*/
ApplyTypeDefs(PdbParser pdbParser, XmlPullParser xmlParser, TaskMonitor monitor,
MessageLog log) throws CancelledException, SAXParseException {
ApplyTypeDefs(PdbParser pdbParser, XmlPullParser xmlParser, TaskMonitor monitor, MessageLog log)
throws CancelledException, SAXParseException {
this.pdbParser = pdbParser;
this.log = log;
@ -99,13 +99,13 @@ class ApplyTypeDefs {
WrappedDataType baseDataType = pdbParser.findDataType(baseDatatypeName);
if (baseDataType == null) {
log.appendMsg("Error: failed to resolve typedef: " + datatypeName + " -> " +
baseDatatypeName);
log.appendMsg("PDB",
"Failed to resolve typedef: " + datatypeName + " -> " + baseDatatypeName);
continue;
}
if (baseDataType.isZeroLengthArray()) {
log.appendMsg(
"Error: zero length array not supported for typedef: " + datatypeName);
log.appendMsg("PDB",
"Zero length array not supported for typedef: " + datatypeName);
continue;
}

View file

@ -272,6 +272,12 @@ class DefaultCompositeMember extends CompositeMember {
}
Structure struct = (Structure) getDataType();
if (struct.isNotYetDefined() && preferredSize > 0) {
// handle special case of empty structure
struct.growStructure(preferredSize);
return;
}
if (struct.getLength() < preferredSize) {
struct.growStructure(preferredSize - struct.getLength());
return;
@ -302,7 +308,18 @@ class DefaultCompositeMember extends CompositeMember {
*/
private void alignComposite(int preferredSize) {
Composite copy = (Composite) memberDataType.copy(dataTypeManager);
// don't attempt to align empty composite - don't complain
if (isStructureContainer()) {
if (structureMemberOffsetMap.isEmpty()) {
return;
}
}
else if (unionMemberList.isEmpty()) {
return;
}
Composite composite = (Composite) memberDataType;
Composite copy = (Composite) composite.copy(dataTypeManager);
int pack = 0;
copy.setPackingValue(pack);
@ -314,12 +331,12 @@ class DefaultCompositeMember extends CompositeMember {
alignOK = isGoodAlignment(copy, preferredSize);
}
if (alignOK) {
((Composite) memberDataType).setPackingValue(pack);
composite.setPackingValue(pack);
}
else if (errorConsumer != null && !isClass) { // don't complain about Class structs which always fail
String anonymousStr = parent != null ? " anonymous " : "";
errorConsumer.accept("PDB " + anonymousStr + memberType +
" reconstruction failed to align " + memberDataType.getPathName());
" reconstruction failed to align " + composite.getPathName());
}
}
@ -1048,7 +1065,8 @@ class DefaultCompositeMember extends CompositeMember {
boolean addMember(DefaultCompositeMember member) {
if (member.memberDataType == null || member.memberDataType.getLength() <= 0) {
Msg.debug(this, "Failed to resolve datatype " + member.getDataTypeName());
Msg.debug(this, "Failed to resolve member datatype for '" + getDataTypeName() + "': " +
member.getDataTypeName());
return false;
}

View file

@ -113,7 +113,7 @@ public class DefaultPdbMember extends PdbMember {
Msg.error(this, "PDB parse error: " + e.getMessage());
return null;
}
wrappedDt = new WrappedDataType(bitFieldDt, false);
wrappedDt = new WrappedDataType(bitFieldDt, false, false);
}
return wrappedDt;
}

View file

@ -26,7 +26,10 @@ import ghidra.util.task.TaskMonitor;
class PdbDataTypeParser {
private final static String NO_TYPE = "NoType";
private final static String NO_TYPE = "<NoType>";
private final static WrappedDataType NO_TYPE_DATATYPE =
new WrappedDataType(new TypedefDataType(NO_TYPE, Undefined1DataType.dataType), false, true);
private DataTypeManager programDataTypeMgr;
private DataTypeManagerService service;
@ -76,12 +79,12 @@ class PdbDataTypeParser {
return programDataTypeMgr;
}
void flushDataTypeCache() {
void flushDataTypeCache(TaskMonitor monitor) throws CancelledException {
for (DataType dt : dataTypeCache.values()) {
monitor.checkCanceled();
programDataTypeMgr.resolve(dt,
DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER);
}
dataTypeCache.clear();
}
/**
@ -111,7 +114,7 @@ class PdbDataTypeParser {
}
void clear() {
dataTypeCache.clear();
dataTypeCache = new HashMap<>();
}
DataType getCachedDataType(String key) {
@ -204,7 +207,7 @@ class PdbDataTypeParser {
}
if (NO_TYPE.equals(datatype)) {
return new WrappedDataType(VoidDataType.dataType, false); //TODO make it void?
return NO_TYPE_DATATYPE;
}
String dataTypeName = datatype;
@ -265,7 +268,7 @@ class PdbDataTypeParser {
dt = createPointer(dt);
}
return new WrappedDataType(dt, isZeroLengthArray);
return new WrappedDataType(dt, isZeroLengthArray, false);
}
private String parseArrayDimensions(String datatype, List<Integer> arrayDimensions) {

View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,10 +15,10 @@
*/
package ghidra.app.util.bin.format.pdb;
import ghidra.app.util.importer.MessageLog;
import org.xml.sax.*;
import ghidra.app.util.importer.MessageLog;
class PdbErrorHandler implements ErrorHandler {
private MessageLog log;
@ -30,24 +29,20 @@ class PdbErrorHandler implements ErrorHandler {
this.log = log;
}
/**
* @see org.xml.sax.ErrorHandler#error(org.xml.sax.SAXParseException)
*/
@Override
public void error(SAXParseException exception) throws SAXException {
if (log != null) log.appendMsg(exception.getMessage());
if (log != null)
log.appendMsg("PDB XML Error: " + exception.getMessage());
}
/**
* @see org.xml.sax.ErrorHandler#fatalError(org.xml.sax.SAXParseException)
*/
@Override
public void fatalError(SAXParseException exception) throws SAXException {
if (log != null) log.appendMsg(exception.getMessage());
if (log != null)
log.appendMsg("PDB XML Error: " + exception.getMessage());
}
/**
* @see org.xml.sax.ErrorHandler#warning(org.xml.sax.SAXParseException)
*/
@Override
public void warning(SAXParseException exception) throws SAXException {
log.appendMsg(exception.getMessage());
log.appendMsg("PDB XML Warning: " + exception.getMessage());
}
}
}

View file

@ -289,12 +289,12 @@ public class PdbParser {
}
if (applyTypeDefs != null) {
applyTypeDefs.buildTypeDefs(monitor); // TODO: no dependencies exit on TypeDefs (use single pass)
applyTypeDefs.buildTypeDefs(monitor); // TODO: no dependencies exist on TypeDefs (use single pass)
}
// Ensure that all data types are resolved
if (dataTypeParser != null) {
dataTypeParser.flushDataTypeCache();
dataTypeParser.flushDataTypeCache(monitor);
}
}
@ -326,9 +326,7 @@ public class PdbParser {
if (hasErrors()) {
throw new IOException(getErrorAndWarningMessages());
}
if (monitor.isCancelled()) {
return;
}
monitor.checkCanceled();
XmlElement element = parser.next();
if (!element.isStart()) {
continue;
@ -392,7 +390,7 @@ public class PdbParser {
options.setBoolean(PdbParserConstants.PDB_LOADED, true);
if (dataTypeParser != null && dataTypeParser.hasMissingBitOffsetError()) {
log.error("PDB Parser",
log.error("PDB",
"One or more bitfields were specified without bit-offset data.\nThe use of old pdb.xml data could be the cause.");
}
}
@ -431,6 +429,7 @@ public class PdbParser {
private void defineClasses(MessageLog log) throws CancelledException {
// create namespace and classes in an ordered fashion use tree map
monitor.setMessage("Define classes...");
monitor.initialize(namespaceMap.size());
for (SymbolPath path : namespaceMap.keySet()) {
monitor.checkCanceled();
@ -439,7 +438,7 @@ public class PdbParser {
NamespaceUtils.getNamespace(program, path.getParent(), null);
if (parentNamespace == null) {
String type = isClass ? "class" : "namespace";
log.appendMsg("Error: failed to define " + type + ": " + path);
log.appendMsg("PDB", "Failed to define " + type + ": " + path);
continue;
}
defineNamespace(parentNamespace, path.getName(), isClass, log);
@ -467,8 +466,9 @@ public class PdbParser {
else if (namespace.getSymbol().getSymbolType() == SymbolType.NAMESPACE) {
return;
}
log.appendMsg("Unable to create class namespace due to conflicting symbol: " +
namespace.getName(true));
log.appendMsg("PDB",
"Unable to create class namespace due to conflicting symbol: " +
namespace.getName(true));
}
else if (isClass) {
symbolTable.createClass(parentNamespace, name, SourceType.IMPORTED);
@ -478,8 +478,8 @@ public class PdbParser {
}
}
catch (Exception e) {
log.appendMsg("Unable to create class namespace: " + parentNamespace.getName(true) +
Namespace.NAMESPACE_DELIMITER + name);
log.appendMsg("PDB", "Unable to create class namespace: " +
parentNamespace.getName(true) + Namespace.NAMESPACE_DELIMITER + name);
}
}
@ -739,6 +739,7 @@ public class PdbParser {
EnumDataType createEnum(String name, int length) {
SymbolPath path = new SymbolPath(name);
length = Integer.max(length, 1);
return new EnumDataType(getCategory(path.getParent(), true), path.getName(), length,
dataMgr);
}
@ -751,7 +752,7 @@ public class PdbParser {
void createData(Address address, String datatype, MessageLog log) throws CancelledException {
WrappedDataType wrappedDt = getDataTypeParser().findDataType(datatype);
if (wrappedDt == null) {
log.appendMsg("Error: Failed to resolve datatype " + datatype + " at " + address);
log.appendMsg("PDB", "Failed to resolve datatype " + datatype + " at " + address);
}
else if (wrappedDt.isZeroLengthArray()) {
Msg.debug(this, "Did not apply zero length array data " + datatype + " at " + address);
@ -765,8 +766,8 @@ public class PdbParser {
DumbMemBufferImpl memBuffer = new DumbMemBufferImpl(program.getMemory(), address);
DataTypeInstance dti = DataTypeInstance.getDataTypeInstance(dataType, memBuffer);
if (dti == null) {
log.appendMsg(
"Error: Failed to apply datatype " + dataType.getName() + " at " + address);
log.appendMsg("PDB",
"Failed to apply datatype " + dataType.getName() + " at " + address);
}
else {
createData(address, dti.getDataType(), dti.getLength(), log);
@ -781,7 +782,7 @@ public class PdbParser {
CodeUnit cu = program.getListing().getCodeUnitContaining(address);
if (cu != null) {
if ((cu instanceof Instruction) || !address.equals(cu.getAddress())) {
log.appendMsg("Warning: Did not create data type \"" + dataType.getDisplayName() +
log.appendMsg("PDB", "Did not create data type \"" + dataType.getDisplayName() +
"\" at address " + address + " due to conflict");
return;
}
@ -795,8 +796,8 @@ public class PdbParser {
return;
}
if (dataType.getLength() <= 0 && dataTypeLength <= 0) {
log.appendMsg("Unknown dataTypeLength specified at address " + address + " for " +
dataType.getName());
log.appendMsg("PDB", "Unknown dataTypeLength specified at address " + address +
" for " + dataType.getName());
return;
}
@ -828,8 +829,8 @@ public class PdbParser {
}
}
catch (Exception e) {
log.appendMsg("Unable to create " + dataType.getDisplayName() + " at 0x" + address +
": " + e.getMessage());
log.appendMsg("PDB", "Unable to create " + dataType.getDisplayName() + " at 0x" +
address + ": " + e.getMessage());
}
}
else if (isDataReplaceable(existingData)) {
@ -838,7 +839,7 @@ public class PdbParser {
listing.createData(address, dataType, dataTypeLength);
}
catch (Exception e) {
log.appendMsg("Unable to replace " + dataType.getDisplayName() + " at 0x" +
log.appendMsg("PDB", "Unable to replace " + dataType.getDisplayName() + " at 0x" +
address + ": " + e.getMessage());
}
}
@ -846,9 +847,9 @@ public class PdbParser {
DataType existingDataType = existingData.getDataType();
String existingDataTypeString =
existingDataType == null ? "null" : existingDataType.getDisplayName();
log.appendMsg("Warning: Did not create data type \"" + dataType.getDisplayName() +
"\" at address " + address + ". Preferring existing datatype \"" +
existingDataTypeString + "\"");
log.appendMsg("PDB",
"Did not create data type \"" + dataType.getDisplayName() + "\" at address " +
address + ". Preferring existing datatype \"" + existingDataTypeString + "\"");
}
}
@ -952,7 +953,7 @@ public class PdbParser {
return true;
}
catch (InvalidInputException e) {
log.appendMsg("Unable to create symbol: " + e.getMessage());
log.appendMsg("PDB", "Unable to create symbol at " + address + ": " + e.getMessage());
}
return false;
}
@ -1374,7 +1375,7 @@ public class PdbParser {
PdbXmlMember(XmlElement element) {
super(SymbolUtilities.replaceInvalidChars(element.getAttribute("name"), false),
element.getAttribute("datatype"),
SymbolUtilities.replaceInvalidChars(element.getAttribute("datatype"), false),
XmlUtilities.parseInt(element.getAttribute("offset")),
PdbKind.parse(element.getAttribute("kind")), getDataTypeParser());
}

View file

@ -125,7 +125,7 @@ final class PdbUtil {
// }
// }
// else if (actualLength > expectedLength) {
// log.appendMsg("Warning: Composite data type generated from PDB has size mismatch. " +
// log.appendMsg("PDB", "Composite data type generated from PDB has size mismatch. " +
// composite.getName() + ": expected 0x" + Integer.toHexString(expectedLength) +
// ", but was 0x" + Integer.toHexString(actualLength));
// }

View file

@ -28,6 +28,7 @@ import ghidra.program.model.data.DataType;
public class WrappedDataType {
private final boolean isZeroLengthArray;
private final boolean isNoType;
private final DataType dataType;
/**
@ -36,10 +37,13 @@ public class WrappedDataType {
* @param isZeroLengthArray true if datatype corresponds to a zero-length
* array which can not directly be represented as an Array datatype,
* else false for all other cases.
* @param isNoType if true wrapped type corresponds to NoType as
* used by PDB forced to have a size of 1-byte.
*/
protected WrappedDataType(DataType dataType, boolean isZeroLengthArray) {
protected WrappedDataType(DataType dataType, boolean isZeroLengthArray, boolean isNoType) {
this.dataType = dataType;
this.isZeroLengthArray = isZeroLengthArray;
this.isNoType = isNoType;
}
/**
@ -57,4 +61,12 @@ public class WrappedDataType {
public boolean isZeroLengthArray() {
return isZeroLengthArray;
}
/**
* @return true if wrapped type corresponds to NoType as
* used by PDB forced to have a size of 1-byte.
*/
public boolean isNoType() {
return isNoType;
}
}

View file

@ -124,9 +124,9 @@ void iterateDataTypes(PDBApiContext& ctx) {
}
const ULONGLONG len = getLength(*pSymbol);
if (len == 0) {
continue;
}
// if (len == 0) {
// continue;
// }
printf("%S<datatype name=\"%S\" kind=\"%S\" length=\"0x%I64x\" >\n", indent(8).c_str(), getName(*pSymbol).c_str(), getKindAsString(*pSymbol).c_str(), len);
@ -192,9 +192,9 @@ void iterateClasses(PDBApiContext& ctx) {
}
const ULONGLONG len = getLength(*pSymbol);
if ( len == 0 ) {
continue;
}
// if ( len == 0 ) {
// continue;
// }
printf("%S<class name=\"%S\" length=\"0x%I64x\" >\n", indent(8).c_str(), getName(*pSymbol).c_str(), len);
@ -323,7 +323,7 @@ void dumpFunctionLines( IDiaSymbol& symbol, IDiaSession& session )
DWORD end = 0;
pLine->get_lineNumberEnd( &end );
printf("%S<line_number source_file=\"%ws\" start=\"0x%x\" end=\"0x%x\" addr=\"0x%x\" /> \n",
printf("%S<line_number source_file=\"%ws\" start=\"%d\" end=\"%d\" addr=\"0x%x\" /> \n",
indent(12).c_str(), sourceFileName.GetBSTR(), start, end, addr);
}
}