GP-1036 EXTERNAL Block Offset-Reference refactor

This commit is contained in:
ghidra1 2022-03-08 05:08:02 -05:00
parent 332480a422
commit 2bced1cd43
43 changed files with 461 additions and 216 deletions

View file

@ -2115,8 +2115,9 @@ public class CodeManager implements ErrorHandler, ManagerDB {
long pointerComponentOffset =
PointerTypedefInspector.getPointerComponentOffset((TypeDef) dataType);
if (pointerComponentOffset != 0) {
refManager.addOffsetMemReference(data.getMinAddress(), toAddr,
pointerComponentOffset, RefType.DATA, SourceType.DEFAULT, 0);
refManager.addOffsetMemReference(data.getMinAddress(),
toAddr.subtractWrap(pointerComponentOffset), true, pointerComponentOffset,
RefType.DATA, SourceType.DEFAULT, 0);
return;
}
}

View file

@ -31,7 +31,6 @@ import ghidra.program.model.address.*;
import ghidra.program.model.data.*;
import ghidra.program.model.lang.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.model.pcode.HighFunction;
import ghidra.program.model.symbol.*;
import ghidra.program.model.util.PropertyMapManager;
@ -348,8 +347,7 @@ public class FunctionManagerDB implements FunctionManager {
if (body == null || !body.contains(entryPoint)) {
throw new IllegalArgumentException("Function body must contain the entrypoint");
}
if (codeMgr.getDefinedDataAt(entryPoint) != null &&
!MemoryBlock.isExternalBlockAddress(entryPoint, program)) {
if (codeMgr.getDefinedDataAt(entryPoint) != null) {
throw new IllegalArgumentException(
"Function entryPoint may not be created on defined data");
}

View file

@ -57,6 +57,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
private AddressSet initializedLoadedAddrSet = new AddressSet();
private AddressSet allInitializedAddrSet = new AddressSet();
private AddressSetView executeSet = null;
private AddressSet externalBlockAddrSet;
private MemoryBlock lastBlock;// the last accessed block
private LiveMemoryHandler liveMemory;
@ -130,6 +131,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
addrSet = new AddressSet();
initializedLoadedAddrSet = new AddressSet();
allInitializedAddrSet = new AddressSet();
externalBlockAddrSet = new AddressSet();
// we have to process the non-mapped blocks first because to process the mapped
// blocks we need the address sets for the non-mapped blocks to be complete
for (MemoryBlockDB block : blocks) {
@ -161,6 +163,10 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
private void addBlockAddresses(MemoryBlockDB block, boolean scanAllMappedBlocksIfNeeded) {
AddressSet blockSet = new AddressSet(block.getStart(), block.getEnd());
addrSet = addrSet.union(blockSet);
if (block.isExternalBlock()) {
// NOTE: no handling for mapped blocks which should never map onto EXTERNAL block
externalBlockAddrSet.add(block.getStart(), block.getEnd());
}
if (block.isMapped()) {
// Identify source-blocks which block maps onto and add as a mapped-block to each of these
@ -1765,6 +1771,11 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
return addrSet.isEmpty();
}
@Override
public boolean isExternalBlockAddress(Address addr) {
return externalBlockAddrSet.contains(addr);
}
@Override
public Address getMinAddress() {
return addrSet.getMinAddress();

View file

@ -15,11 +15,10 @@
*/
package ghidra.program.database.references;
import ghidra.program.model.address.*;
import ghidra.program.model.listing.*;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Program;
import ghidra.program.model.symbol.*;
import ghidra.program.util.*;
import ghidra.util.task.TaskMonitor;
import ghidra.program.util.SimpleDiffUtility;
class MemReferenceDB extends ReferenceDB {
@ -44,6 +43,10 @@ class MemReferenceDB extends ReferenceDB {
this(program, from, to, type, opIndex, sourceType, isPrimary, symbolID, false, false, 0);
}
protected boolean isExternalBlockReference() {
return program.getMemory().isExternalBlockAddress(getToAddress());
}
public boolean isOffset() {
return isOffset;
}
@ -56,12 +59,6 @@ class MemReferenceDB extends ReferenceDB {
return offsetOrShift;
}
/**
* @see ghidra.program.database.references.ReferenceDB#setProgram(ghidra.program.model.listing.Program)
*/
void setProgram(Program program) {
}
/**
* @see ghidra.program.model.symbol.Reference#isOffsetReference()
*/

View file

@ -1,6 +1,5 @@
/* ###
* IP: GHIDRA
* REVIEWED: YES
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -28,17 +27,17 @@ class OffsetReferenceDB extends MemReferenceDB implements OffsetReference {
offset);
}
/**
* @see ghidra.program.model.symbol.OffsetReference#getOffset()
*/
@Override
public long getOffset() {
return offsetOrShift;
}
/**
* @see ghidra.program.model.symbol.OffsetReference#getBaseAddress()
*/
@Override
public Address getBaseAddress() {
if (isExternalBlockReference()) {
// EXTERNAL block: must report toAddr and base as the same regardless of offset
return toAddr;
}
return toAddr.subtractWrap(offsetOrShift);
}
@ -56,4 +55,5 @@ class OffsetReferenceDB extends MemReferenceDB implements OffsetReference {
OffsetReference ref = (OffsetReference) obj;
return offsetOrShift == ref.getOffset();
}
}

View file

@ -34,6 +34,7 @@ import ghidra.program.model.pcode.Varnode;
import ghidra.program.model.symbol.*;
import ghidra.program.util.ChangeManager;
import ghidra.util.Lock;
import ghidra.util.Msg;
import ghidra.util.exception.*;
import ghidra.util.task.TaskMonitor;
@ -486,12 +487,48 @@ public class ReferenceDBManager implements ReferenceManager, ManagerDB, ErrorHan
return null;
}
private boolean isExternalBlockAddress(Address addr) {
return program.getMemory().isExternalBlockAddress(addr);
}
@Override
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, long offset,
RefType type, SourceType sourceType, int opIndex) {
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, boolean toAddrIsBase,
long offset, RefType type, SourceType sourceType, int opIndex) {
if (!fromAddr.isMemoryAddress() || !toAddr.isMemoryAddress()) {
throw new IllegalArgumentException("From and To addresses must be memory addresses");
}
// Handle EXTERNAL Block offset-reference transformation
boolean isExternalBlockRef = isExternalBlockAddress(toAddr);
boolean badOffsetReference = false;
if (isExternalBlockRef) {
// NOTE: Resulting EXTERNAL Block reference may become incorrect
// if EXTERNAL block is moved or removed.
if (!toAddrIsBase) {
Address baseAddr = toAddr.subtractWrap(offset);
if (isExternalBlockAddress(baseAddr)) {
toAddr = baseAddr;
toAddrIsBase = true;
}
else {
// assume unintentional reference into EXTERNAL block
isExternalBlockRef = false;
badOffsetReference = true;
}
}
}
else if (toAddrIsBase) {
Address addr = toAddr.addWrap(offset);
if (isExternalBlockAddress(addr)) {
badOffsetReference = true;
}
}
if (badOffsetReference) {
Msg.warn(this, "Offset Reference from " + fromAddr +
" produces bad Xref into EXTERNAL block");
}
try {
removeNonMemRefs(fromAddr, opIndex);
return addRef(fromAddr, toAddr, type, sourceType, opIndex, true, false, offset);
@ -1883,7 +1920,9 @@ public class ReferenceDBManager implements ReferenceManager, ManagerDB, ErrorHan
Reference memRef;
if (ref.isOffsetReference()) {
OffsetReference offRef = (OffsetReference) ref;
memRef = addOffsetMemReference(from, to, offRef.getOffset(), type, sourceType, opIndex);
memRef =
addOffsetMemReference(from, offRef.getBaseAddress(), true, offRef.getOffset(), type,
sourceType, opIndex);
}
else if (ref.isShiftedReference()) {
ShiftedReference shiftRef = (ShiftedReference) ref;

View file

@ -23,6 +23,7 @@ import ghidra.framework.store.LockException;
import ghidra.program.database.mem.*;
import ghidra.program.model.address.*;
import ghidra.program.model.listing.Program;
import ghidra.program.model.symbol.OffsetReference;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.NotFoundException;
import ghidra.util.task.TaskMonitor;
@ -87,6 +88,23 @@ public interface Memory extends AddressSetView {
*/
public boolean isBigEndian();
/**
* Determine if the specified address is contained within the reserved EXTERNAL block
* (see {@link MemoryBlock#EXTERNAL_BLOCK_NAME}). This artificial memory block has certain
* limitations that may require associated addresses to be properly identified. All
* data access/referencing has the biggest exposure since the importers generally
* allocate a fixed and possibly insufficient amount of memory to corresponding data
* symbols. Any pointer math performed based upon an EXTERNAL block symbol address
* is likely to produce an unuseable address that may collide with unrelated symbols
* stored within the memory block (e.g., {@link OffsetReference} is one such example).
* @param addr address
* @return true if address is contained within EXTERNAL memory block, else false.
*/
public default boolean isExternalBlockAddress(Address addr) {
MemoryBlock block = getBlock(addr);
return block != null && block.isExternalBlock();
}
/**
* Sets the live memory handler
* @param handler the live memory handler

View file

@ -22,7 +22,8 @@ import java.util.List;
import ghidra.framework.store.LockException;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Program;
import ghidra.program.model.address.AddressSpace;
import ghidra.program.model.symbol.OffsetReference;
import ghidra.util.NamingUtilities;
/**
@ -31,8 +32,18 @@ import ghidra.util.NamingUtilities;
public interface MemoryBlock extends Serializable, Comparable<MemoryBlock> {
/**
* A special EXTERNAL block may be created by certain program loaders (e.g., Elf) to act as a
* stand-in for unknown external symbol locations.
* A special purpose EXTERNAL block may be created by certain program loaders
* (e.g., Elf) to act as a stand-in for unknown external symbol locations when
* relocation support is required using a valid memory address. While the
* EXTERNAL block is created out of neccessity for relocation processing it
* introduces a number of limitations when used to carry data symbols
* where pointer math and offset-references may occur.
* <p>
* The method {@link Memory#isExternalBlockAddress(Address)}
* may be used to determine if a specific address is contained within an EXTERNAL memory block.
* <p>
* NOTE: Close proximity to the end of an address space should be avoided
* to allow for {@link OffsetReference} use.
*/
public static final String EXTERNAL_BLOCK_NAME = "EXTERNAL";
@ -272,9 +283,27 @@ public interface MemoryBlock extends Serializable, Comparable<MemoryBlock> {
/**
* Returns true if this is either a bit-mapped or byte-mapped block
*
* @return true if this is either a bit-mapped or byte-mapped block
*/
public boolean isMapped();
/**
* Returns true if this is a reserved EXTERNAL memory block based upon its name
* (see {@link MemoryBlock#EXTERNAL_BLOCK_NAME}). Checks for individual addresses may be done
* using {@link Memory#isExternalBlockAddress(Address)}.
* <p>
* Note that EXTERNAL blocks always resides within a memory space and never within the artifial
* {@link AddressSpace#EXTERNAL_SPACE} which is not a memory space. This can be a source of confusion.
* An EXTERNAL memory block exists to facilitate relocation processing for some external
* symbols which require a real memory address.
*
* @return true if this is a reserved EXTERNAL memory block
*/
public default boolean isExternalBlock() {
return EXTERNAL_BLOCK_NAME.equals(getName());
}
/**
* Returns true if this is an overlay block (i.e., contained within overlay space).
*
@ -300,19 +329,4 @@ public interface MemoryBlock extends Serializable, Comparable<MemoryBlock> {
*/
public List<MemoryBlockSourceInfo> getSourceInfos();
/**
* Determine if the specified address is contained within the reserved EXTERNAL block.
*
* @param address address of interest
* @param program
* @return true if address is contained within the reserved EXTERNAL block, else false.
*/
public static boolean isExternalBlockAddress(Address address, Program program) {
Memory memory = program.getMemory();
if (!address.isMemoryAddress()) {
return false;
}
MemoryBlock block = memory.getBlock(address);
return block != null && MemoryBlock.EXTERNAL_BLOCK_NAME.equals(block.getName());
}
}

View file

@ -16,10 +16,15 @@
package ghidra.program.model.symbol;
import ghidra.program.model.address.Address;
import ghidra.program.model.mem.MemoryBlock;
/**
* <code>OffsetReference</code> is a memory reference whose "to" address is
* computed from a base address plus an offset.
* <p>
* NOTE: References into the reserved EXTERNAL block must report {@link #getToAddress()}
* the same as {@link #getBaseAddress()} regardless of offset value due to symbol
* spacing limitations within the EXTERNAL block. See {@link MemoryBlock#EXTERNAL_BLOCK_NAME}.
*/
public interface OffsetReference extends Reference {
@ -35,4 +40,12 @@ public interface OffsetReference extends Reference {
*/
public Address getBaseAddress();
/**
* Return the base address plus the offset. The exception to this is the
* EXTERNAL block case where this method returns the base address regardless
* of the offset value.
* @return reference "to" address
*/
public Address getToAddress();
}

View file

@ -18,6 +18,7 @@ package ghidra.program.model.symbol;
import ghidra.program.model.address.*;
import ghidra.program.model.lang.Register;
import ghidra.program.model.listing.Variable;
import ghidra.program.model.mem.MemoryBlock;
import ghidra.util.exception.DuplicateNameException;
import ghidra.util.exception.InvalidInputException;
@ -32,7 +33,8 @@ public interface ReferenceManager {
/**
* Add a memory, stack, register or external reference
* @param reference
* @param reference reference to be added
* @return new reference
*/
public Reference addReference(Reference reference);
@ -45,6 +47,7 @@ public interface ReferenceManager {
* @param stackOffset stack offset of the reference
* @param type reference type - how the location is being referenced.
* @param source the source of this reference
* @return new stack reference
*/
public Reference addStackReference(Address fromAddr, int opIndex, int stackOffset, RefType type,
SourceType source);
@ -58,12 +61,13 @@ public interface ReferenceManager {
* @param register register to add the reference to
* @param type reference type - how the location is being referenced.
* @param source the source of this reference
* @return new register reference
*/
public Reference addRegisterReference(Address fromAddr, int opIndex, Register register,
RefType type, SourceType source);
/**
* Adds a memory reference. Only first the first memory reference placed on
* Adds a memory reference. The first memory reference placed on
* an operand will be made primary by default. All non-memory references
* will be removed from the specified operand.
* @param fromAddr address of the codeunit where the reference occurs
@ -73,37 +77,46 @@ public interface ReferenceManager {
* @param source the source of this reference
* @param opIndex the operand index
* display of the operand making this reference
* @return new memory reference
*/
public Reference addMemoryReference(Address fromAddr, Address toAddr, RefType type,
SourceType source, int opIndex);
/**
* Add an offset memory reference. Only first the first memory reference placed on
* Add an offset memory reference. The first memory reference placed on
* an operand will be made primary by default. All non-memory references
* will be removed from the specified operand.
* will be removed from the specified operand. If toAddr corresponds to
* the EXTERNAL memory block (see {@link MemoryBlock#EXTERNAL_BLOCK_NAME}) the
* resulting offset reference will report to/base address as the same
* regardless of specified offset.
* @param fromAddr address for the "from"
* @param toAddr address of the "to"
* @param toAddr address of the location being referenced.
* @param toAddrIsBase if true toAddr is treated as base address, else treated as (base+offet).
* It is generally preferred to specify as a base address to ensure proper handling of
* EXTERNAL block case.
* @param offset value added to a base address to get the toAddr
* @param type reference type - how the location is being referenced
* @param source the source of this reference
* @param opIndex the operand index
* @return new offset reference
*/
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, long offset,
RefType type, SourceType source, int opIndex);
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, boolean toAddrIsBase,
long offset, RefType type, SourceType source, int opIndex);
/**
* Add a shifted memory reference; the "to" address is computed as the value
* at the operand at opIndex shifted by some number of bits, specified in the
* shiftValue parameter. Only first the first memory reference placed on
* shiftValue parameter. The first memory reference placed on
* an operand will be made primary by default. All non-memory references
* will be removed from the specified operand.
* @param fromAddr address for the "from"
* @param toAddr computed as the value of the operand at opIndex shifted
* by the number of bits specified by shiftValue
* @param shiftValue
* @param shiftValue shifted value
* @param type reference type - how the location is being referenced
* @param source the source of this reference
* @param opIndex the operand index
* @return new shifted reference
*/
public Reference addShiftedMemReference(Address fromAddr, Address toAddr, int shiftValue,
RefType type, SourceType source, int opIndex);
@ -117,8 +130,9 @@ public interface ReferenceManager {
* @param extLabel label within the external program, may be null if extAddr is not null
* @param extAddr address within the external program, may be null
* @param source the source of this reference
* @param type reference type - how the location is being referenced
* @param opIndex operand index
* @param type reference type - how the location is being referenced
* @return new external space reference
* @throws InvalidInputException
* @throws DuplicateNameException
*/
@ -135,8 +149,9 @@ public interface ReferenceManager {
* @param extLabel label within the external program, may be null if extAddr is not null
* @param extAddr address within the external program, may be null
* @param source the source of this reference
* @param type reference type - how the location is being referenced
* @param opIndex operand index
* @param type reference type - how the location is being referenced
* @return new external space reference
* @throws InvalidInputException
* @throws DuplicateNameException
*/
@ -191,7 +206,7 @@ public interface ReferenceManager {
/**
* Returns the referenced function variable.
* @param reference
* @param reference variable reference
* @return function variable or null if variable not found
*/
public Variable getReferencedVariable(Reference reference);
@ -204,19 +219,23 @@ public interface ReferenceManager {
public void setPrimary(Reference ref, boolean isPrimary);
/**
* Return whether the given address has flow references from this address.
* Return whether the given address has flow references from it.
* @param addr the address to test for flow references.
* @return true if the given address has flow references from it, else false
*/
public boolean hasFlowReferencesFrom(Address addr);
/**
* Get the flow references from the given address.
* Get all flow references from the given address.
* @param addr the address of the codeunit to get all flows from.
* @return get all flow references from the given address.
*
*/
public Reference[] getFlowReferencesFrom(Address addr);
/**
* Returns an iterator over all external references
* Returns an iterator over all external space references
* @return reference iterator over all external space references
*/
public ReferenceIterator getExternalReferences();
@ -224,6 +243,7 @@ public interface ReferenceManager {
* Get an iterator over all references that have the given address as
* their "To" address.
* @param addr the address that all references in the iterator refer to.
* @return reference iterator over all references to the specified address.
*/
public ReferenceIterator getReferencesTo(Address addr);
@ -242,6 +262,7 @@ public interface ReferenceManager {
* @param fromAddr the address of the codeunit making the reference.
* @param toAddr the address being referred to.
* @param opIndex the operand index.
* @return reference which satisfies the specified criteria or null
*/
public Reference getReference(Address fromAddr, Address toAddr, int opIndex);
@ -267,6 +288,8 @@ public interface ReferenceManager {
* fromAddr.
* @param fromAddr the address of the codeunit being tested
* @param opIndex the index of the operand being tested.
* @return true if one or more reference from the specified address
* and opindex are defined, else false
*/
public boolean hasReferencesFrom(Address fromAddr, int opIndex);
@ -274,6 +297,8 @@ public interface ReferenceManager {
* Returns true if there are any memory references at the given
* address.
* @param fromAddr the address of the codeunit being tested
* @return true if one or more reference from the specified address
* are defined, else false
*/
public boolean hasReferencesFrom(Address fromAddr);
@ -281,6 +306,8 @@ public interface ReferenceManager {
* Get the primary reference from the given address.
* @param addr from address
* @param opIndex operand index
* @return the primary reference from the specified address
* and opindex if it exists, else null
*/
public Reference getPrimaryReferenceFrom(Address addr, int opIndex);
@ -289,6 +316,7 @@ public interface ReferenceManager {
* reference
* @param startAddr address to position iterator.
* @param forward true means to iterate in the forward direction
* @return address iterator where references from exist
*/
public AddressIterator getReferenceSourceIterator(Address startAddr, boolean forward);
@ -297,6 +325,8 @@ public interface ReferenceManager {
* reference, restricted by the given address set.
* @param addrSet the set of address to restrict the iterator or null for all addresses.
* @param forward true means to iterate in the forward direction
* address iterator where references from exist
* @return address iterator where references from exist constrained by addrSet
*/
public AddressIterator getReferenceSourceIterator(AddressSetView addrSet, boolean forward);
@ -305,6 +335,8 @@ public interface ReferenceManager {
* reference.
* @param startAddr start of iterator
* @param forward true means to iterate in the forward direction
* address iterator where references to exist
* @return address iterator where references to exist
*/
public AddressIterator getReferenceDestinationIterator(Address startAddr, boolean forward);
@ -313,20 +345,21 @@ public interface ReferenceManager {
* memory reference, restricted by the given address set.
* @param addrSet the set of address to restrict the iterator or null for all addresses.
* @param forward true means to iterate in the forward direction
* @return address iterator where references to exist constrained by addrSet
*/
public AddressIterator getReferenceDestinationIterator(AddressSetView addrSet, boolean forward);
/**
* Returns the number of memory References to the specified
* <code>toAddr</code>
* Returns the number of references to the specified <code>toAddr</code>.
* @param toAddr the address being referenced
* @return the number of references to the specified <code>toAddr</code>.
*/
public int getReferenceCountTo(Address toAddr);
/**
* Returns the number of memory References from the specified
* <code>fromAddr</code>
* Returns the number of references from the specified <code>fromAddr</code>.
* @param fromAddr the address of the codeunit making the reference.
* @return the number of references from the specified <code>fromAddr</code>.
*/
public int getReferenceCountFrom(Address fromAddr);
@ -343,6 +376,7 @@ public interface ReferenceManager {
/**
* Return true if a memory reference exists with the given "to" address.
* @param toAddr address being referred to.
* @return true if specified toAddr has one or more references to it, else false.
*/
public boolean hasReferencesTo(Address toAddr);
@ -350,6 +384,7 @@ public interface ReferenceManager {
* Uodate the reference type on a memory reference.
* @param ref reference to be updated
* @param refType new reference type
* @return updated reference
*/
public Reference updateRefType(Reference ref, RefType refType);
@ -379,6 +414,7 @@ public interface ReferenceManager {
/**
* Returns the reference level for the references to the given address
* @param toAddr the address at which to find the highest reference level
* @return reference level for specified to address.
*/
public byte getReferenceLevel(Address toAddr);

View file

@ -48,8 +48,8 @@ public class ReferenceManagerTestDouble implements ReferenceManager {
}
@Override
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, long offset,
RefType type, SourceType source, int opIndex) {
public Reference addOffsetMemReference(Address fromAddr, Address toAddr, boolean toAddrIsBase,
long offset, RefType type, SourceType source, int opIndex) {
throw new UnsupportedOperationException();
}