Merge remote-tracking branch

'origin/GP-1778_ghidravore_address_range_map_iterator--SQUASHED'

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/ProgramDB.java

@@ -200,7 +200,6 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 	private int languageMinorVersion;
 	private LanguageTranslator languageUpgradeTranslator;
 
-	private Address storedImageBase; // current image base maintained by addrMap
 	private boolean imageBaseOverride = false;
 	private boolean recordChanges;
 
@@ -1119,7 +1118,7 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 
 	private void refreshImageBase() throws IOException {
 		long baseOffset = getStoredBaseImageOffset();
-		storedImageBase = addressFactory.getDefaultAddressSpace().getAddress(baseOffset);
+		Address storedImageBase = addressFactory.getDefaultAddressSpace().getAddress(baseOffset);
 		if (!imageBaseOverride) {
 			Address currentImageBase = getImageBase();
 			if (!currentImageBase.equals(storedImageBase)) {
@@ -1268,7 +1267,6 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 			if (commit) {
 				try {
 					dataMap.put(IMAGE_OFFSET, Long.toHexString(base.getOffset()));
-					storedImageBase = base;
 					imageBaseOverride = false;
 
 					setChanged(ChangeManager.DOCR_IMAGE_BASE_CHANGED, oldBase, base);
@@ -1736,9 +1734,7 @@ public class ProgramDB extends DomainObjectAdapterDB implements Program, ChangeM
 			refreshName();
 			overlaySpaceAdapter.updateOverlaySpaces(addressFactory);
 			addrMap.invalidateCache();
-			if (!imageBaseOverride) {
-				refreshImageBase();
-			}
+			refreshImageBase();
 			for (int i = 0; i < NUM_MANAGERS; i++) {
 				managers[i].invalidateCache(all);
 			}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMap.java

@@ -87,12 +87,19 @@ public interface AddressMap {
 	 * never be generated.  The returned key ranges will correspond 
 	 * to those key ranges which have previously been created within 
 	 * the specified address range and may represent a much smaller subset 
-	 * of addresses within the specified range.
-	 * @param start minimum address of range
+	 * of addresses within the specified range. 
+	 * NOTE: if the create parameter is true, the given range must not extend in the upper 32 bits 
+	 * by more than 1 segment. For example, range(0x0000000000000000 - 0x0000000100000000) 
+	 * is acceptable, but the range (0x0000000000000000 - 0x0000000200000000) is not because the
+	 * upper 32 bits differ by 2.
+	 * @param start the start address of the range
 	 * @param end maximum address of range
 	 * @param create true if a new keys may be generated, otherwise returned 
-	 * key-ranges will be limited to those already defined.
+	 * key-ranges will be limited to those already defined. And if true, the range will be limited
+	 * to a size of 2^32 so that at most it creates two new address bases
 	 * @return "sorted" list of KeyRange objects
+	 * @throws UnsupportedOperationException if the given range is so large that the upper 32 bit
+	 * segments differ by more than 1.
 	 */
 	public List<KeyRange> getKeyRanges(Address start, Address end, boolean create);
 
@@ -136,7 +143,8 @@ public interface AddressMap {
 	 * key-ranges will be limited to those already defined.
 	 * @return "sorted" list of KeyRange objects
 	 */
-	public List<KeyRange> getKeyRanges(Address start, Address end, boolean absolute, boolean create);
+	public List<KeyRange> getKeyRanges(Address start, Address end, boolean absolute,
+			boolean create);
 
 	/**
 	 * Generates a properly ordered list of database key ranges for a

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/map/AddressMapDB.java

@@ -15,6 +15,8 @@
  */
 package ghidra.program.database.map;
 
+import static generic.util.UnsignedDataUtils.*;
+
 import java.io.IOException;
 import java.util.*;
 
@@ -811,15 +813,16 @@ public class AddressMapDB implements AddressMap {
 	/**
 	 * Create all memory base segments within the specified range.
 	 * NOTE: minAddress and maxAddress must have the same address space!
-	 * @param minAddress
-	 * @param maxAddress
+	 * @param minAddress start address of the range
+	 * @param maxAddress last address of the range
+	 * @param absolute if the address are absolute and not relative
 	 */
-	private void createBaseSegments(Address minAddress, Address maxAddress) {
+	private void createBaseSegments(Address minAddress, Address maxAddress, boolean absolute) {
 
 		long minBase;
 		long maxBase;
 
-		if (isInDefaultAddressSpace(minAddress)) {
+		if (!absolute && isInDefaultAddressSpace(minAddress)) {
 			minBase = getNormalizedOffset(minAddress) & BASE_MASK;
 			maxBase = getNormalizedOffset(maxAddress) & BASE_MASK;
 		}
@@ -828,7 +831,15 @@ public class AddressMapDB implements AddressMap {
 			maxBase = maxAddress.getOffset() & BASE_MASK;
 		}
 
-		for (long base = minBase; base <= maxBase; base += (MAX_OFFSET + 1)) {
+		long numBases = (maxBase >>> 32) - (minBase >>> 32);
+
+		if (numBases > 2) {
+			throw new UnsupportedOperationException("Can't create address bases for a range that" +
+				"extends across more than two upper 32 bit segments!");
+		}
+
+		for (long base = minBase; unsignedLessThanOrEqual(base, maxBase); base +=
+			(MAX_OFFSET + 1)) {
 			getBaseAddressIndex(minAddress.getNewAddress(base), false, INDEX_CREATE);
 		}
 	}
@@ -836,17 +847,19 @@ public class AddressMapDB implements AddressMap {
 	/**
 	 * Add simple key ranges where the address range lies within a single base segment for a single space.
 	 * NOTE: start and end addresses must have the same address space!
-	 * @param keyRangeList
-	 * @param start
-	 * @param end
-	 * @param absolute
-	 * @param create
+	 * @param keyRangeList the list to store key ranges into
+	 * @param start the start address
+	 * @param end the end address
+	 * @param absolute true if the address are to be encoded as absolute (not relative to the
+	 *  image base
+	 * @param create if true, this method will add new address bases that are required to
+	 * store addresses in that database that have that address base
 	 */
 	private void addKeyRanges(List<KeyRange> keyRangeList, Address start, Address end,
 			boolean absolute, boolean create) {
 		if (start.isMemoryAddress()) {
 			if (create) {
-				createBaseSegments(start, end);
+				createBaseSegments(start, end, absolute);
 			}
 			Address normalizedStart = absolute ? start : getShiftedAddr(start);
 			Address normalizedEnd = absolute ? end : getShiftedAddr(end);

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/register/AddressRangeObjectMap.java

@@ -15,12 +15,12 @@
  */
 package ghidra.program.database.register;
 
+import java.util.*;
+
 import ghidra.program.model.address.*;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
-import java.util.*;
-
 /**
  * Associates objects with address ranges.
  */
@@ -170,7 +170,8 @@ public class AddressRangeObjectMap<T> {
 			T object, AddressValueRange<T> newRange, int pos) {
 
 		AddressValueRange<T> previousRange = ranges.get(pos);
-		if ((start.previous() == null) || (previousRange.getEnd().compareTo(start.previous()) < 0)) {
+		if ((start.previous() == null) ||
+			(previousRange.getEnd().compareTo(start.previous()) < 0)) {
 			return newRange;  // no overlap
 		}
 
@@ -469,6 +470,10 @@ public class AddressRangeObjectMap<T> {
 		}
 		return new AddressRangeImpl(min, max);
 	}
+
+	public void clearCache() {
+		lastRange = null;
+	}
 }
 
 /**

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/register/DatabaseRangeMapAdapter.java

@@ -209,4 +209,9 @@ public class DatabaseRangeMapAdapter implements RangeMapAdapter {
 		}
 	}
 
+	@Override
+	public void invalidate() {
+		rangeMap.invalidate();
+	}
+
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/register/InMemoryRangeMapAdapter.java

@@ -1,6 +1,5 @@
 /* ###
  * IP: GHIDRA
- * REVIEWED: YES
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -143,4 +142,9 @@ public class InMemoryRangeMapAdapter implements RangeMapAdapter {
 		}
 		rangeMap = newRangeMap;
 	}
+
+	@Override
+	public void invalidate() {
+		rangeMap.clearCache();
+	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/register/ProgramRegisterContextDB.java

@@ -175,6 +175,7 @@ public class ProgramRegisterContextDB extends AbstractStoredProgramContext imple
 	@Override
 	public void invalidateCache(boolean all) throws IOException {
 		this.invalidateReadCache();
+		invalidateRegisterStores();
 	}
 
 	@Override
@@ -525,4 +526,12 @@ public class ProgramRegisterContextDB extends AbstractStoredProgramContext imple
 		}
 	}
 
+	private void invalidateRegisterStores() {
+		for (RegisterValueStore store : registerValueMap.values()) {
+			store.invalidate();
+		}
+		for (RegisterValueStore store : defaultRegisterValueMap.values()) {
+			store.invalidate();
+		}
+	}
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/util/AddressRangeMapIterator.java

@@ -0,0 +1,281 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.program.database.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import db.DBRecord;
+import db.RecordIterator;
+import ghidra.program.database.map.AddressKeyRecordIterator;
+import ghidra.program.database.map.AddressMap;
+import ghidra.program.model.address.*;
+import ghidra.util.Lock;
+
+/**
+ * An iterator over ranges that have a defined values in the AddressRangeMapDB
+ * <P>
+ * NOTE: this iterator is complicated by the fact that there can exist a record that represents
+ * an address range that "wraps around" from the max address to the 0 address, where this record
+ * actually represents two address ranges. This is cause by changing the image base which shifts
+ * all records up or down. That shift can cause a record to have a wrapping range where the start
+ * address is larger than the end address. If such a record exists, it is found during construction
+ * and the lower address range is extracted from the record and is stored as a special "start range"
+ * that should be emitted before any other ranges in that space. The upper range of a wrapping
+ * record will be handled more naturally during the iteration process. When a wrapping record is
+ * encountered during the normal iteration, only the upper range is used and it will be in the
+ * correct address range ordering.
+ */
+class AddressRangeMapIterator implements AddressRangeIterator {
+
+	private AddressRangeMapDB rangeMap;
+	private AddressMap addressMap;
+	private Lock lock;
+	private int expectedModCount;
+
+	private DBRecord nextRecord;
+	private RecordIterator recordIterator;
+
+	// this is the range from a wrapping address record that needs to be emitted before any other
+	// ranges in the default space. It is discovered during construction if it exists
+	private AddressRange startRange;
+
+	// these will be null if iterating over all records
+	private Address iteratorStart;
+	private Address iteratorEnd;
+
+	/**
+	 * Constructs an AddressRangeIterator over all ranges that have a defined value
+	 * @param rangeMap the AddressRangeMapDB to iterate over
+	 * @throws IOException if a database I/O exception occurs
+	 */
+	AddressRangeMapIterator(AddressRangeMapDB rangeMap) throws IOException {
+		this.rangeMap = rangeMap;
+		this.lock = rangeMap.getLock();
+		this.addressMap = rangeMap.getAddressMap();
+		this.expectedModCount = rangeMap.getModCount();
+		this.recordIterator = new AddressKeyRecordIterator(rangeMap.getTable(), addressMap);
+		startRange = checkForStartRangeFromWrappingAddressRecord();
+	}
+
+	/**
+	 * Constructs an AddressRangeIterator over all ranges greater than the given start address
+	 * that have a defined value. If start is in the middle of a defined range, the first
+	 * range will be truncated to start with the given start address.
+	 * @param rangeMap the AddressRangeMapDB to iterate over
+	 * @param start the address where the iterator should start
+	 * @throws IOException if a database I/O exception occurs
+	 */
+	AddressRangeMapIterator(AddressRangeMapDB rangeMap, Address start) throws IOException {
+		this(rangeMap, start, null);
+	}
+
+	/**
+	 * Constructs an AddressRangeIterator over all ranges between the given start address and the
+	 * given end address that have a defined value. If start is in the middle of a defined range,
+	 * the first range returned will truncated to start with the given start address. 
+	 * If the endAddress is in the middle of a defined range, the last range will be truncated to
+	 * end with the given end address
+	 * @param rangeMap the AddressRangeMapDB to iterate over
+	 * @param start the address where the iterator should start
+	 * @param end the address where the iterator should end
+	 * @throws IOException if a database I/O exception occurs
+	 */
+	AddressRangeMapIterator(AddressRangeMapDB rangeMap, Address start, Address end)
+			throws IOException {
+		this.rangeMap = rangeMap;
+		this.lock = rangeMap.getLock();
+		this.addressMap = rangeMap.getAddressMap();
+		this.expectedModCount = rangeMap.getModCount();
+		this.iteratorStart = start;
+		this.iteratorEnd = getIteratorEnd(end);
+
+		// adjust start address to start of previous range if it contains this address
+		// so the iterator will include it
+		AddressRange range = rangeMap.getAddressRangeContaining(iteratorStart);
+		this.recordIterator = new AddressKeyRecordIterator(rangeMap.getTable(), addressMap,
+			range.getMinAddress(), iteratorEnd, range.getMinAddress(), true);
+
+		startRange = trimRange(checkForStartRangeFromWrappingAddressRecord());
+	}
+
+	@Override
+	public Iterator<AddressRange> iterator() {
+		return this;
+	}
+
+	@Override
+	public boolean hasNext() {
+		lock.acquire();
+		try {
+			if (expectedModCount != rangeMap.getModCount()) {
+				throw new ConcurrentModificationException();
+			}
+			if (nextRecord != null) {
+				return true;
+			}
+			if (startRange != null) {
+				return true;
+			}
+			if (recordIterator != null) {
+				try {
+					return recordIterator.hasNext();
+				}
+				catch (IOException e) {
+					rangeMap.dbError(e);
+				}
+			}
+			return false;
+		}
+		finally {
+			lock.release();
+		}
+	}
+
+	@Override
+	public AddressRange next() {
+		lock.acquire();
+		try {
+			if (expectedModCount != rangeMap.getModCount()) {
+				throw new ConcurrentModificationException();
+			}
+			if (nextRecord != null) {
+				DBRecord record = nextRecord;
+				nextRecord = null;
+				return getAddressRange(record);
+			}
+			if (recordIterator == null || !recordIterator.hasNext()) {
+				if (startRange != null) {
+					// there are no more items in the underlying iterator, so if we have a 
+					// discovered start range, as described in the class header, then return that
+					AddressRange range = startRange;
+					startRange = null;
+					return range;
+				}
+				return null;
+			}
+			return getAddressRange(recordIterator.next());
+		}
+		catch (IOException e) {
+			rangeMap.dbError(e);
+			return null;
+		}
+		finally {
+			lock.release();
+		}
+	}
+
+	/**
+	 * Computes an address range for the given record. If a wrapping record (a record whose start
+	 * address is greater than its end address, so it really represents two ranges) is encountered,
+	 * it only returns the high range. The low range is specially found in the constructor and 
+	 * returned before any other ranges in that same address space are returned.
+	 * @param record the record to convert to an address range
+	 * @return the address range represented by the given record
+	 */
+	private AddressRange getAddressRange(DBRecord record) {
+		List<AddressRange> ranges = rangeMap.getRangesForRecord(record);
+		if (ranges.isEmpty()) {
+			return null;
+		}
+
+		// Normally, there is only one range for a record. If it is an address wrapping record
+		// caused by an image base change, then the record will produce two ranges. While iterating
+		// we always want the last (higher) range. 
+		AddressRange range = ranges.get(ranges.size() - 1);
+
+		// if there is a "start range" discovered in the constructor and this is the first
+		// range we see in the default space (the space with image base), then we need to return
+		// the start range and save this record for next time.
+		if (startRange != null && hasSameSpace(startRange.getMinAddress(), range.getMinAddress())) {
+			range = startRange;
+			startRange = null;
+
+			// save current record into next record so we process it again next time
+			nextRecord = record;
+		}
+
+		return trimRange(range);
+	}
+
+	private boolean hasSameSpace(Address address1, Address address2) {
+		AddressSpace space1 = address1.getAddressSpace();
+		AddressSpace space2 = address2.getAddressSpace();
+		return space1.equals(space2);
+	}
+
+	/**
+	 * Look for a start range that needs to be issued before any other ranges in the default
+	 * space.
+	 * <P> 
+	 * Because of a changed image base, it is possible that a single record can represent
+	 * multiple ranges, as described in the class header. This is the code that will look for that
+	 * case and correct the ordering.
+	 *  
+	 * @return the start range that needs to be issued before any other ranges in this space. 
+	 * @throws IOException if a database I/O error occurs
+	 */
+	private AddressRange checkForStartRangeFromWrappingAddressRecord()
+			throws IOException {
+
+		DBRecord record = rangeMap.getAddressWrappingRecord();
+		if (record == null) {
+			return null;
+		}
+
+		List<AddressRange> ranges = rangeMap.getRangesForRecord(record);
+		// we want the lower range - the upper range will be handle later during iteration
+		return ranges.get(0);
+	}
+
+	private Address getIteratorEnd(Address end) {
+		if (end != null) {
+			return end;	// non-null end was supplied, use that
+		}
+		AddressFactory factory = addressMap.getAddressFactory();
+		AddressSet allAddresses = factory.getAddressSet();
+		return allAddresses.getMaxAddress();
+	}
+
+	/**
+	 * Make sure the range is within the iterator's given start and end range. This really only
+	 * matters for the first and last range returned by the iterator, but it hard to know when
+	 * the given range is the first or last, just just trim all returned ranges.
+	 * @param range the range to be trimmed
+	 * @return the trimmed address range
+	 */
+	private AddressRange trimRange(AddressRange range) {
+		if (range == null) {
+			return null;
+		}
+		// if no iterator bounds set, no trimming needed
+		if (iteratorStart == null) {
+			return range;
+		}
+
+		if (range.compareTo(iteratorStart) > 0 && range.compareTo(iteratorEnd) < 0) {
+			return range;
+		}
+
+		Address start = Address.max(range.getMinAddress(), iteratorStart);
+		Address end = Address.min(range.getMaxAddress(), iteratorEnd);
+		if (start.compareTo(end) <= 0) {
+			return new AddressRangeImpl(start, end);
+		}
+		return null;
+	}
+
+}

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/listing/ProgramContext.java

@@ -243,4 +243,5 @@ public interface ProgramContext {
 	 * @return disassembly context register value
 	 */
 	public RegisterValue getDisassemblyContext(Address address);
+
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/RangeMapAdapter.java

@@ -109,4 +109,9 @@ public interface RangeMapAdapter {
 	 */
 	public void checkWritableState();
 
+	/**
+	 * Notification that something has changed that may affect internal caching
+	 */
+	public void invalidate();
+
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/util/RegisterValueStore.java

@@ -15,14 +15,14 @@
  */
 package ghidra.program.util;
 
+import java.util.*;
+
 import ghidra.program.model.address.*;
 import ghidra.program.model.lang.Register;
 import ghidra.program.model.lang.RegisterValue;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
-import java.util.*;
-
 /**
  * This is a generalized class for storing register values over ranges.  The values include mask bits
  * to indicate which bits within the register are being set.  The mask is stored along with the
@@ -324,4 +324,11 @@ public class RegisterValueStore {
 		return rangeMap.getValueRangeContaining(addr);
 	}
 
+	/**
+	 * Notifies that something changed, may need to invalidate any caches
+	 */
+	public void invalidate() {
+		rangeMap.invalidate();
+	}
+
 }