From db27e3cba03eefb76708dce576e250d5eba07c2e Mon Sep 17 00:00:00 2001 From: ghidra1 Date: Wed, 27 Apr 2022 10:55:05 -0400 Subject: [PATCH] GP-1949 - MemoryMap, AddressMap and ELF MemorySectionResolver import performance improvements --- .../util/opinion/MemorySectionResolver.java | 32 +- .../program/database/map/AddressMapDB.java | 8 + .../program/database/mem/MemoryBlockDB.java | 3 + .../program/database/mem/MemoryMapDB.java | 516 ++++++++++++------ .../database/mem/MemoryMapDBAdapterV3.java | 31 +- .../java/ghidra/program/model/mem/Memory.java | 2 + 6 files changed, 382 insertions(+), 210 deletions(-) diff --git a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/MemorySectionResolver.java b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/MemorySectionResolver.java index d3f3e78529..e75fa57076 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/MemorySectionResolver.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/MemorySectionResolver.java @@ -35,6 +35,7 @@ public abstract class MemorySectionResolver { protected final Program program; private Set usedBlockNames = new HashSet<>(); + private AddressSet physicalLoadedOverlaySet; private List sections = new ArrayList<>(); // built-up prior to resolve private Map sectionIndexMap = new HashMap<>(); @@ -267,6 +268,16 @@ public abstract class MemorySectionResolver { // build-up mapping of sections to a sequence of memory ranges sectionMemoryMap = new HashMap<>(); + physicalLoadedOverlaySet = new AddressSet(); + for (MemoryBlock block : getMemory().getBlocks()) { + Address minAddr = block.getStart(); + Address maxAddr = block.getEnd(); + if (minAddr.isLoadedMemoryAddress() && minAddr.getAddressSpace().isOverlaySpace()) { + physicalLoadedOverlaySet.add(minAddr.getPhysicalAddress(), + maxAddr.getPhysicalAddress()); + } + } + // process sections in reverse order - last-in takes precedence int sectionCount = sections.size(); monitor.initialize(sectionCount); @@ -390,6 +401,10 @@ public abstract class MemorySectionResolver { minAddr = block.getStart(); maxAddr = block.getEnd(); usedBlockNames.add(blockName); + if (block.isOverlay() && minAddr.isLoadedMemoryAddress()) { + physicalLoadedOverlaySet.add(minAddr.getPhysicalAddress(), + maxAddr.getPhysicalAddress()); + } } else { // block may be null due to unexpected conflict or pruning - allow to continue @@ -437,19 +452,10 @@ public abstract class MemorySectionResolver { } // Get base memory conflict set - Memory memory = getMemory(); - AddressSet rangeSet = new AddressSet(rangeMin, rangeMax); - AddressSet conflictSet = memory.intersect(rangeSet); - - // Add in loaded overlay conflicts (use their physical address) - for (MemoryBlock block : memory.getBlocks()) { - Address minAddr = block.getStart(); - Address maxAddr = block.getEnd(); - if (minAddr.isLoadedMemoryAddress() && minAddr.getAddressSpace().isOverlaySpace()) { - AddressSet intersection = rangeSet.intersectRange(minAddr.getPhysicalAddress(), - maxAddr.getPhysicalAddress()); - conflictSet.add(intersection); - } + AddressSet conflictSet = getMemory().intersectRange(rangeMin, rangeMax); + if (!physicalLoadedOverlaySet.isEmpty()) { + // Add in loaded overlay physical address conflicts + conflictSet.add(physicalLoadedOverlaySet.intersectRange(rangeMin, rangeMax)); } return conflictSet; diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMapDB.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMapDB.java index d4ef092158..f16a8f5859 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMapDB.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMapDB.java @@ -99,6 +99,8 @@ public class AddressMapDB implements AddressMap { private Address[] sortedBaseEndAddrs; private List allKeyRanges; // all existing key ranges (includes non-absolute memory, and external space) private HashMap addrToIndexMap = new HashMap(); + private Address lastBaseAddress; + private int lastIndex; private long baseImageOffset; // pertains to default address space only private List segmentedRanges; // when using segmented memory, this list contains @@ -253,6 +255,7 @@ public class AddressMapDB implements AddressMap { @Override public synchronized void invalidateCache() throws IOException { + lastBaseAddress = null; if (!readOnly) { baseAddrs = adapter.getBaseAddresses(true); init(true); @@ -391,8 +394,13 @@ public class AddressMapDB implements AddressMap { AddressSpace space = addr.getAddressSpace(); Address tBase = space.getAddressInThisSpaceOnly(normalizedBaseOffset); + if (tBase.equals(lastBaseAddress)) { + return lastIndex; + } Integer tIndex = addrToIndexMap.get(tBase); if (tIndex != null) { + lastBaseAddress = tBase; + lastIndex = tIndex; return tIndex; } else if (indexOperation == INDEX_MATCH) { diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryBlockDB.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryBlockDB.java index 16fc4b3635..0fef32edca 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryBlockDB.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryBlockDB.java @@ -107,6 +107,7 @@ public class MemoryBlockDB implements MemoryBlock { * @return collection of blocks which map onto this block or null if none identified */ Collection getMappedBlocks() { + memMap.buildAddressSets(); // updates mappedBlocks if needed return mappedBlocks; } @@ -261,6 +262,7 @@ public class MemoryBlockDB implements MemoryBlock { try { checkValid(); setPermissionBit(EXECUTE, x); + memMap.blockExecuteChanged(this); memMap.fireBlockChanged(this); } finally { @@ -276,6 +278,7 @@ public class MemoryBlockDB implements MemoryBlock { setPermissionBit(READ, read); setPermissionBit(WRITE, write); setPermissionBit(EXECUTE, execute); + memMap.blockExecuteChanged(this); memMap.fireBlockChanged(this); } finally { diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDB.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDB.java index 0b2d8c48fa..d261ed6924 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDB.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDB.java @@ -53,11 +53,21 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { private DataConverter defaultEndian; private List blocks;// sorted list of blocks - private AddressSet addrSet = new AddressSet(); - private AddressSet initializedLoadedAddrSet = new AddressSet(); - private AddressSet allInitializedAddrSet = new AddressSet(); - private AddressSetView executeSet = null; - private AddressSet externalBlockAddrSet; + private AddressSet allAddrSet; // continuously updated + + private MemoryAddressSetView addrSetView; + + /** + * Address set views into program memory which are iterator safe + * for public API methods. + */ + private class MemoryAddressSetView { + private AddressSet all = new AddressSet(); + private AddressSet initializedAndLoaded = new AddressSet(); + private AddressSet initialized = new AddressSet(); + private AddressSet externalBlock = new AddressSet(); + private AddressSet execute = new AddressSet(); // may be replaced when block permissions change + } private MemoryBlock lastBlock;// the last accessed block private LiveMemoryHandler liveMemory; @@ -127,111 +137,83 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { } } - private void buildAddressSets() { - addrSet = new AddressSet(); - initializedLoadedAddrSet = new AddressSet(); - allInitializedAddrSet = new AddressSet(); - externalBlockAddrSet = new AddressSet(); - // we have to process the non-mapped blocks first because to process the mapped - // blocks we need the address sets for the non-mapped blocks to be complete - for (MemoryBlockDB block : blocks) { - block.clearMappedBlockList(); - if (!block.isMapped()) { - addBlockAddresses(block, false); + void buildAddressSets() { + lock.acquire(); + try { + // null addrSet signals address sets must be built otherwise return + if (addrSetView != null) { + return; + } + addrSetView = new MemoryAddressSetView(); + + // we have to process the non-mapped blocks first because to process the mapped + // blocks we need the address sets for the non-mapped blocks to be complete + for (MemoryBlockDB block : blocks) { + block.clearMappedBlockList(); + if (!block.isMapped()) { + addBlockAddresses(block); + } + } + // process all mapped blocks after non-mapped-blocks above + for (MemoryBlockDB block : blocks) { + if (block.isMapped()) { + addBlockAddresses(block); + } + } + + if (allAddrSet == null) { + // An independent address set copy is maintained which may be changed + // dynamically when adding memory blocks. This is neccessary + // since such a changing address set may not be used to hand out + // iterators. Any time allAddrSet is modified addrSetView is + // set to null to signal it becoming stale. + allAddrSet = new AddressSet(addrSetView.all); } } - // process all mapped blocks after non-mapped-blocks above - for (MemoryBlockDB block : blocks) { - if (block.isMapped()) { - addBlockAddresses(block, false); - } + finally { + lock.release(); } } /** - * Update the allInitializedAddrSet and initializedLoadedAddrSet with + * Update the addrSetView.initialized and addrSetView.initializedAndLoaded with * relevant initialized addresses from the specified memory block. If block is not a * mapped-block and it may be a source to existing mapped-blocks then * scanAllMappedBlocksIfNeeded should be passed as true unless all * mapped blocks will be processed separately. * * @param block memory block - * @param scanAllMappedBlocksIfNeeded if true and block is initialized and not a mapped block + * @param blockMayBeMappedOnto if true and block is initialized and not a mapped block * all mapped blocks will be processed for possible introduction of newly initialized * mapped regions. */ - private void addBlockAddresses(MemoryBlockDB block, boolean scanAllMappedBlocksIfNeeded) { - AddressSet blockSet = new AddressSet(block.getStart(), block.getEnd()); - addrSet = addrSet.union(blockSet); + private void addBlockAddresses(MemoryBlockDB block) { + Address start = block.getStart(); + Address end = block.getEnd(); + addrSetView.all.add(start, end); if (block.isExternalBlock()) { - // NOTE: no handling for mapped blocks which should never map onto EXTERNAL block - externalBlockAddrSet.add(block.getStart(), block.getEnd()); + addrSetView.externalBlock.add(start, end); + } + else if (block.isExecute()) { + addrSetView.execute.add(start, end); } if (block.isMapped()) { - // Identify source-blocks which block maps onto and add as a mapped-block to each of these AddressRange mappedRange = block.getSourceInfos().get(0).getMappedRange().get(); for (MemoryBlockDB b : getBlocks(mappedRange.getMinAddress(), mappedRange.getMaxAddress())) { - b.addMappedBlock(block); + if (!b.isMapped()) { + b.addMappedBlock(block); + } } - - AddressSet mappedSet = getMappedIntersection(block, allInitializedAddrSet); - allInitializedAddrSet = allInitializedAddrSet.union(mappedSet); - initializedLoadedAddrSet = initializedLoadedAddrSet.union( - getMappedIntersection(block, initializedLoadedAddrSet)); + AddressSet mappedSet = getMappedIntersection(block, addrSetView.initialized); + addrSetView.initialized.add(mappedSet); + addrSetView.initializedAndLoaded.add(getMappedIntersection(block, addrSetView.initializedAndLoaded)); } else if (block.isInitialized()) { - allInitializedAddrSet = allInitializedAddrSet.union(blockSet); + addrSetView.initialized.add(block.getStart(), block.getEnd()); if (block.isLoaded()) { - initializedLoadedAddrSet = initializedLoadedAddrSet.union(blockSet); - } - if (scanAllMappedBlocksIfNeeded) { - // If only adding one initialized non-mapped-block we must scan all mapped-blocks - // which may utilize block as a byte source - for (MemoryBlockDB b : blocks) { - b.clearMappedBlockList(); - } - for (MemoryBlockDB b : blocks) { - if (b.isMapped()) { - addBlockAddresses(b, false); - } - } - } - } - } - - /** - * Update initialized address set for those mapped blocks which map onto the specified block - * which has just completed a transition of its' initialized state. - * - * @param block block whose initialized state has changed - * @param isInitialized true if block transitioned from uninitialized to initialized, else - * transition is from initialized to uninitialized. - */ - private void updateMappedAddresses(MemoryBlockDB block, boolean isInitialized) { - - Collection mappedBlocks = block.getMappedBlocks(); - if (mappedBlocks == null) { - return; - } - - AddressSet blockSet = new AddressSet(block.getStart(), block.getEnd()); - boolean isLoaded = block.getStart().isLoadedMemoryAddress(); - - for (MemoryBlockDB mappedBlock : block.getMappedBlocks()) { - AddressSet mappedSet = getMappedIntersection(mappedBlock, blockSet); - if (isInitialized) { - allInitializedAddrSet = allInitializedAddrSet.union(mappedSet); - if (isLoaded) { - initializedLoadedAddrSet = initializedLoadedAddrSet.union(mappedSet); - } - } - else { - allInitializedAddrSet = allInitializedAddrSet.subtract(mappedSet); - if (isLoaded) { - initializedLoadedAddrSet = initializedLoadedAddrSet.union(mappedSet); - } + addrSetView.initializedAndLoaded.add(block.getStart(), block.getEnd()); } } } @@ -240,6 +222,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { synchronized (this) { fileBytesAdapter.refresh(); adapter.refreshMemory(); + allAddrSet = null; initializeBlocks(); buildAddressSets(); } @@ -250,14 +233,29 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { } private synchronized void initializeBlocks() { - List newBlocks = adapter.getMemoryBlocks(); + blocks = adapter.getMemoryBlocks(); lastBlock = null; - blocks = newBlocks; nameBlockMap = new HashMap<>(); - executeSet = null; + addrSetView = null; // signal stale view addrMap.memoryMapChanged(this); } + void blockExecuteChanged(MemoryBlockDB block) { + // lock must be active + if (addrSetView == null) { + return; + } + // copy must be made to remain iterator safe + AddressSet set = new AddressSet(addrSetView.execute); + if (block.isExecute()) { + set.addRange(block.getStart(), block.getEnd()); + } + else { + set.deleteRange(block.getStart(), block.getEnd()); + } + addrSetView.execute = set; + } + public void setLanguage(Language newLanguage) { defaultEndian = newLanguage.isBigEndian() ? BIG_ENDIAN : LITTLE_ENDIAN; } @@ -316,17 +314,48 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { return getLoadedAndInitializedAddressSet(); } + private AddressSetView getIterableAddressSet() { + lock.acquire(); + try { + if (addrSetView == null) { + buildAddressSets(); + } + return new AddressSetViewAdapter(addrSetView.all); + } + finally { + lock.release(); + } + } + @Override public AddressSetView getAllInitializedAddressSet() { - return new AddressSetViewAdapter(allInitializedAddrSet); + lock.acquire(); + try { + if (addrSetView == null) { + buildAddressSets(); + } + return new AddressSetViewAdapter(addrSetView.initialized); + } + finally { + lock.release(); + } } @Override public AddressSetView getLoadedAndInitializedAddressSet() { if (liveMemory != null) { - return this;//all memory is initialized! + return this; // all memory is initialized! + } + lock.acquire(); + try { + if (addrSetView == null) { + buildAddressSets(); + } + return new AddressSetViewAdapter(addrSetView.initializedAndLoaded); + } + finally { + lock.release(); } - return new AddressSetViewAdapter(initializedLoadedAddrSet); } void checkMemoryWrite(MemoryBlockDB block, Address start, long length) @@ -514,9 +543,6 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { // name could have changed nameBlockMap = new HashMap<>(); - - // don't regenerate now, do lazily later if needed - executeSet = null; } void fireBytesChanged(Address addr, int count) { @@ -618,8 +644,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { MemoryBlockDB newBlock = adapter.createInitializedBlock(name, start, is, length, MemoryBlock.READ); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, !overlay); fireBlockAdded(newBlock); return newBlock; } @@ -662,8 +688,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { MemoryBlockDB newBlock = adapter.createFileBytesBlock(name, start, length, fileBytes, offset, MemoryBlock.READ); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, !overlay); fireBlockAdded(newBlock); return newBlock; } @@ -715,8 +741,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { MemoryBlockDB newBlock = adapter.createBlock(MemoryBlockType.DEFAULT, name, start, size, null, false, MemoryBlock.READ, 0); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, false); fireBlockAdded(newBlock); return newBlock; } @@ -750,8 +776,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { MemoryBlockDB newBlock = adapter.createBlock(MemoryBlockType.BIT_MAPPED, name, start, length, mappedAddress, false, MemoryBlock.READ, 0); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, false); fireBlockAdded(newBlock); return newBlock; } @@ -794,8 +820,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { MemoryBlockDB newBlock = adapter.createBlock(MemoryBlockType.BYTE_MAPPED, name, start, length, mappedAddress, false, MemoryBlock.READ, mappingScheme); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, false); fireBlockAdded(newBlock); return newBlock; } @@ -842,8 +868,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { } MemoryBlockDB newBlock = adapter.createBlock(block.getType(), name, start, length, mappedAddr, block.isInitialized(), block.getPermissions(), mappingScheme); + allAddrSet.add(newBlock.getStart(), newBlock.getEnd()); initializeBlocks(); - addBlockAddresses(newBlock, !block.isMapped() && block.isInitialized()); fireBlockAdded(newBlock); return newBlock; } @@ -860,7 +886,13 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public long getSize() { - return addrSet.getNumAddresses(); + lock.acquire(); + try { + return allAddrSet.getNumAddresses(); + } + finally { + lock.release(); + } } @Override @@ -901,8 +933,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { // the code manager will be locked until the remove is done try { Address newEndAddr = newStartAddr.addNoWrap(block.getSize() - 1); - AddressSet set = new AddressSet(addrSet); - set.deleteRange(block.getStart(), block.getEnd()); + AddressSet set = new AddressSet(allAddrSet); // could be slow + set.delete(block.getStart(), block.getEnd()); if (set.intersects(newStartAddr, newEndAddr)) { throw new MemoryConflictException( "Block move conflicts with other existing memory block"); @@ -1006,7 +1038,6 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { reloadAll(); newBlock = getBlockDB(block1Addr); fireBlocksJoined(newBlock, block2Addr); - } catch (IOException e) { program.dbError(e); @@ -1084,12 +1115,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { MemoryBlockDB memBlock = (MemoryBlockDB) unitializedBlock; try { memBlock.initializeBlock(initialValue); - allInitializedAddrSet.addRange(memBlock.getStart(), memBlock.getEnd()); - initializedLoadedAddrSet.addRange(memBlock.getStart(), memBlock.getEnd()); - if (!memBlock.isMapped()) { - // update initialized sets for all blocks mapped to memBlock - updateMappedAddresses(memBlock, true); - } + initializeBlocks(); fireBlockChanged(memBlock); fireBytesChanged(memBlock.getStart(), (int) memBlock.getSize()); return memBlock; @@ -1125,12 +1151,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { // FIXME: clear instructions in initializedBlock or any block which maps to it memBlock.uninitializeBlock(); - allInitializedAddrSet.deleteRange(memBlock.getStart(), memBlock.getEnd()); - initializedLoadedAddrSet.deleteRange(memBlock.getStart(), memBlock.getEnd()); - if (!memBlock.isMapped()) { - // update initialized sets for all blocks mapped to memBlock - updateMappedAddresses(memBlock, false); - } + initializeBlocks(); fireBlockChanged(memBlock); fireBytesChanged(memBlock.getStart(), (int) memBlock.getSize()); return memBlock; @@ -1153,8 +1174,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { if (monitor == null) { monitor = TaskMonitor.DUMMY; } - - AddressIterator it = initializedLoadedAddrSet.getAddresses(addr, forward); + AddressSetView set = getLoadedAndInitializedAddressSet(); + AddressIterator it = set.getAddresses(addr, forward); byte[] b = new byte[bytes.length]; if (forward) { while (it.hasNext() && !monitor.isCancelled()) { @@ -1164,7 +1185,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { Address jumpAddr = addr2.addNoWrap(-moffset); if (jumpAddr.hasSameAddressSpace(addr2)) { - it = initializedLoadedAddrSet.getAddresses(jumpAddr, forward); + it = set.getAddresses(jumpAddr, forward); } monitor.incrementProgress(-moffset); } @@ -1200,7 +1221,8 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { if (monitor == null) { monitor = TaskMonitor.DUMMY; } - AddressIterator it = allInitializedAddrSet.getAddresses(startAddr, forward); + AddressSetView set = getLoadedAndInitializedAddressSet(); + AddressIterator it = set.getAddresses(startAddr, forward); byte[] b = new byte[bytes.length]; if (forward) { while (it.hasNext() && !monitor.isCancelled()) { @@ -1213,7 +1235,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { try { Address jumpAddr = addr2.addNoWrap(-moffset); if (jumpAddr.hasSameAddressSpace(addr2)) { - it = allInitializedAddrSet.getAddresses(jumpAddr, forward); + it = set.getAddresses(jumpAddr, forward); } monitor.incrementProgress(-moffset); } @@ -1753,47 +1775,98 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public boolean contains(Address addr) { - return addrSet.contains(addr); + lock.acquire(); + try { + return allAddrSet.contains(addr); + } + finally { + lock.release(); + } } @Override public boolean contains(Address start, Address end) { - return addrSet.contains(start, end); + lock.acquire(); + try { + return allAddrSet.contains(start, end); + } + finally { + lock.release(); + } } @Override - public boolean contains(AddressSetView set) { - return addrSet.contains(set); + public boolean contains(AddressSetView s) { + lock.acquire(); + try { + return allAddrSet.contains(s); + } + finally { + lock.release(); + } } @Override public boolean isEmpty() { - return addrSet.isEmpty(); + lock.acquire(); + try { + return allAddrSet.isEmpty(); + } + finally { + lock.release(); + } } @Override public boolean isExternalBlockAddress(Address addr) { - return externalBlockAddrSet.contains(addr); + lock.acquire(); + try { + if (addrSetView == null) { + buildAddressSets(); + } + return addrSetView.externalBlock.contains(addr); + } + finally { + lock.release(); + } } @Override public Address getMinAddress() { - return addrSet.getMinAddress(); + lock.acquire(); + try { + return allAddrSet.getMinAddress(); + } + finally { + lock.release(); + } } @Override public Address getMaxAddress() { - return addrSet.getMaxAddress(); + lock.acquire(); + try { + return allAddrSet.getMaxAddress(); + } + finally { + lock.release(); + } } @Override public int getNumAddressRanges() { - return addrSet.getNumAddressRanges(); + lock.acquire(); + try { + return allAddrSet.getNumAddressRanges(); + } + finally { + lock.release(); + } } @Override public AddressRangeIterator getAddressRanges() { - return addrSet.getAddressRanges(); + return getIterableAddressSet().getAddressRanges(); } @Override @@ -1803,62 +1876,116 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public AddressRangeIterator getAddressRanges(boolean startAtFront) { - return addrSet.getAddressRanges(startAtFront); + return getIterableAddressSet().getAddressRanges(startAtFront); } @Override public long getNumAddresses() { - return addrSet.getNumAddresses(); + lock.acquire(); + try { + return allAddrSet.getNumAddresses(); + } + finally { + lock.release(); + } } @Override public AddressIterator getAddresses(boolean forward) { - return addrSet.getAddresses(forward); + return getIterableAddressSet().getAddresses(forward); } @Override public AddressIterator getAddresses(Address start, boolean forward) { - return addrSet.getAddresses(start, forward); + return getIterableAddressSet().getAddresses(start, forward); } @Override public boolean intersects(AddressSetView set) { - return addrSet.intersects(set); + lock.acquire(); + try { + return allAddrSet.intersects(set); + } + finally { + lock.release(); + } } @Override public boolean intersects(Address start, Address end) { - return addrSet.intersects(start, end); + lock.acquire(); + try { + return allAddrSet.intersects(start, end); + } + finally { + lock.release(); + } } @Override public AddressSet intersect(AddressSetView set) { - return addrSet.intersect(set); + lock.acquire(); + try { + return allAddrSet.intersect(set); + } + finally { + lock.release(); + } } @Override public AddressSet intersectRange(Address start, Address end) { - return addrSet.intersectRange(start, end); + lock.acquire(); + try { + return allAddrSet.intersectRange(start, end); + } + finally { + lock.release(); + } } @Override public AddressSet union(AddressSetView set) { - return addrSet.union(set); + lock.acquire(); + try { + return allAddrSet.union(set); + } + finally { + lock.release(); + } } @Override public AddressSet subtract(AddressSetView set) { - return addrSet.subtract(set); + lock.acquire(); + try { + return allAddrSet.subtract(set); + } + finally { + lock.release(); + } } @Override public AddressSet xor(AddressSetView set) { - return addrSet.xor(set); + lock.acquire(); + try { + return allAddrSet.xor(set); + } + finally { + lock.release(); + } } @Override public boolean hasSameAddresses(AddressSetView set) { - return addrSet.hasSameAddresses(set); + lock.acquire(); + try { + return allAddrSet.hasSameAddresses(set); + } + finally { + lock.release(); + } } @Override @@ -1870,15 +1997,17 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { MemoryBlockDB memBlock = (MemoryBlockDB) block; Address startAddress = block.getStart(); + Address endAddress = block.getEnd(); program.setEventsEnabled(false);// ensure that no domain object change // events go out that would cause screen updates; // the code manager will be locked until the remove is done try { - program.deleteAddressRange(startAddress, memBlock.getEnd(), monitor); + program.deleteAddressRange(startAddress, endAddress, monitor); memBlock.delete(); - reloadAll(); + allAddrSet.delete(startAddress, endAddress); + initializeBlocks(); } catch (IOException e) { program.dbError(e); @@ -1943,7 +2072,7 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { "Block may not span image base address (" + imageBase + ")"); } } - if (addrSet.intersects(start, end)) { + if (allAddrSet.intersects(start, end)) { throw new MemoryConflictException( "Part of range (" + start + ", " + end + ") already exists in memory."); } @@ -2062,19 +2191,25 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public boolean equals(Object obj) { - if (obj instanceof Memory) { - return obj == this; - } - if (obj instanceof AddressSetView) { - lock.acquire(); - try { - return addrSet.equals(obj); + lock.acquire(); + try { + if (obj instanceof Memory) { + return obj == this; } - finally { - lock.release(); + if (obj instanceof AddressSetView) { + lock.acquire(); + try { + return allAddrSet.equals(obj); + } + finally { + lock.release(); + } } + return false; + } + finally { + lock.release(); } - return false; } @Override @@ -2084,28 +2219,12 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public AddressSetView getExecuteSet() { - AddressSetView set = executeSet; - - if (set == null) { - set = computeExecuteSet(); - } - return set; - } - - /** - * @return executable address set - */ - private AddressSetView computeExecuteSet() { lock.acquire(); try { - AddressSet set = new AddressSet(); - for (MemoryBlock block : blocks) { - if (block.isExecute()) { - set.addRange(block.getStart(), block.getEnd()); - } + if (addrSetView == null) { + buildAddressSets(); } - executeSet = new AddressSetViewAdapter(set); - return executeSet; + return new AddressSetViewAdapter(addrSetView.execute); } finally { lock.release(); @@ -2119,37 +2238,61 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public AddressRangeIterator getAddressRanges(Address start, boolean forward) { - return addrSet.getAddressRanges(start, forward); + return getIterableAddressSet().getAddressRanges(start, forward); } @Override public AddressRange getFirstRange() { - return addrSet.getFirstRange(); + lock.acquire(); + try { + return allAddrSet.getFirstRange(); + } + finally { + lock.release(); + } } @Override public AddressRange getLastRange() { - return addrSet.getLastRange(); + lock.acquire(); + try { + return allAddrSet.getLastRange(); + } + finally { + lock.release(); + } } @Override public AddressRange getRangeContaining(Address address) { - return addrSet.getRangeContaining(address); + lock.acquire(); + try { + return allAddrSet.getRangeContaining(address); + } + finally { + lock.release(); + } } @Override public Iterator iterator(boolean forward) { - return addrSet.iterator(forward); + return getIterableAddressSet().iterator(forward); } @Override public Iterator iterator(Address start, boolean forward) { - return addrSet.iterator(start, forward); + return getIterableAddressSet().iterator(start, forward); } @Override public Address findFirstAddressInCommon(AddressSetView set) { - return addrSet.findFirstAddressInCommon(set); + lock.acquire(); + try { + return allAddrSet.findFirstAddressInCommon(set); + } + finally { + lock.release(); + } } @Override @@ -2180,10 +2323,17 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { @Override public FileBytes createFileBytes(String filename, long offset, long size, InputStream is, TaskMonitor monitor) throws IOException, CancelledException { + long oldProgressMax = 0; + long oldProgress = 0; + if (monitor != null) { + oldProgressMax = monitor.getMaximum(); + oldProgress = monitor.getProgress(); + } lock.acquire(); try { if (monitor != null && is != null) { is = new MonitoredInputStream(is, monitor); + monitor.initialize(size); } return fileBytesAdapter.createFileBytes(filename, offset, size, is); } @@ -2192,6 +2342,10 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener { } finally { lock.release(); + if (monitor != null) { + monitor.setMaximum(oldProgressMax); + monitor.setProgress(oldProgress); + } } } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDBAdapterV3.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDBAdapterV3.java index f6cb79360d..1c31013dd4 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDBAdapterV3.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDBAdapterV3.java @@ -76,7 +76,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { private MemoryMapDB memMap; private AddressMapDB addrMap; - private List memoryBlocks = new ArrayList<>(); + private List memoryBlocks = new ArrayList<>(); // sorted list of blocks private long maxSubBlockSize; public MemoryMapDBAdapterV3(DBHandle handle, MemoryMapDB memMap, long maxSubBlockSize, @@ -145,6 +145,14 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { return memoryBlocks; } + private void cacheNewBlock(MemoryBlockDB newBlock) { + int insertionIndex = Collections.binarySearch(memoryBlocks, newBlock); + if (insertionIndex >= 0) { // should not find direct hit + throw new AssertException("New memory block collides with existing block"); + } + memoryBlocks.add(-insertionIndex - 1, newBlock); + } + @Override MemoryBlockDB createInitializedBlock(String name, Address startAddr, InputStream is, long length, int permissions) throws AddressOverflowException, IOException { @@ -169,8 +177,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, subBlocks); - memoryBlocks.add(newBlock); - Collections.sort(memoryBlocks); + cacheNewBlock(newBlock); return newBlock; } catch (IOCancelledException e) { @@ -219,8 +226,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, subBlocks); - memoryBlocks.add(newBlock); - Collections.sort(memoryBlocks); + cacheNewBlock(newBlock); return newBlock; } @@ -237,8 +243,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, subBlocks); - memoryBlocks.add(newBlock); - Collections.sort(memoryBlocks); + cacheNewBlock(newBlock); return newBlock; } @@ -256,11 +261,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, splitBlocks); - int insertionIndex = Collections.binarySearch(memoryBlocks, newBlock); - if (insertionIndex >= 0) { // should not find direct hit - throw new AssertException("New memory block collides with existing block"); - } - memoryBlocks.add(-insertionIndex - 1, newBlock); + cacheNewBlock(newBlock); return newBlock; } @@ -293,8 +294,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, subBlocks); - memoryBlocks.add(newBlock); - Collections.sort(memoryBlocks); + cacheNewBlock(newBlock); return newBlock; } @@ -314,8 +314,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter { memBlockTable.putRecord(blockRecord); MemoryBlockDB newBlock = new MemoryBlockDB(this, blockRecord, subBlocks); - memoryBlocks.add(newBlock); - Collections.sort(memoryBlocks); + cacheNewBlock(newBlock); return newBlock; } diff --git a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/mem/Memory.java b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/mem/Memory.java index eefc954eae..d483e78266 100644 --- a/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/mem/Memory.java +++ b/Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/mem/Memory.java @@ -414,6 +414,7 @@ public interface Memory extends AddressSetView { /** * Finds a sequence of contiguous bytes that match the * given byte array at all bit positions where the mask contains an "on" bit. + * Search is performed over loaded memory only. * * @param addr The beginning address in memory to search. * @param bytes the array of bytes to search for. @@ -432,6 +433,7 @@ public interface Memory extends AddressSetView { * Finds a sequence of contiguous bytes that match the * given byte array at all bit positions where the mask contains an "on" bit. * Starts at startAddr and ends at endAddr. + * Search is performed over loaded memory only. * If forward is true, search starts at startAddr and will end if startAddr ">" endAddr. * If forward is false, search starts at start addr and will end if startAddr "<" endAddr. *