GT-2845 - updated ELF Loader to utilize FileBytes and eliminated its use

of MemoryBlockUtil

Conflicts:
	Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/ElfLoader.java

Ghidra/Features/Base/src/main/java/ghidra/app/util/MemoryBlockUtils.java

@@ -28,6 +28,7 @@ import ghidra.program.model.listing.*;
 import ghidra.program.model.mem.*;
 import ghidra.util.Msg;
 import ghidra.util.exception.DuplicateNameException;
+import ghidra.util.task.TaskMonitor;
 
 /** 
  * Convenience methods for creating memory blocks.
@@ -57,7 +58,44 @@ public class MemoryBlockUtils {
 		try {
 			MemoryBlock block = memory.createUninitializedBlock(name, start, length, isOverlay);
 			setBlockAttributes(block, comment, source, r, w, x);
-			adjustFragment(program, start, name);
+			adjustFragment(program, block.getStart(), name);
+			return block;
+		}
+		catch (LockException e) {
+			log.appendMsg("Failed to create memory block: exclusive lock/checkout required");
+		}
+		catch (Exception e) {
+			log.appendMsg("Failed to create '" + name + "' memory block: " + e.getMessage());
+		}
+		return null;
+	}
+
+	/**
+	 * Create a new initialized memory block.  Initialized to all zeros.
+	 * @param program the program in which to create the block.
+	 * @param isOverlay if true, the block will be created in a new overlay space for that block
+	 * @param name the name of the new block.
+	 * @param start the starting address of the new block.
+	 * @param is source of the data used to fill the block or null for zero initialization.
+	 * @param length the length of the new block
+	 * @param comment the comment text to associate with the new block.
+	 * @param source the source of the block (This field is not well defined - currently another comment)
+	 * @param r the read permission for the new block.
+	 * @param w the write permission for the new block.
+	 * @param x the execute permission for the new block.
+	 * @param log a {@link MessageLog} for appending error messages
+	 * @return the newly created block or null if the operation failed.
+	 */
+	public static MemoryBlock createInitializedBlock(Program program, boolean isOverlay,
+			String name, Address start, long length, String comment, String source, boolean r,
+			boolean w, boolean x, MessageLog log) {
+
+		Memory memory = program.getMemory();
+		try {
+			MemoryBlock block = memory.createInitializedBlock(name, start, null, length,
+				TaskMonitor.DUMMY, isOverlay);
+			setBlockAttributes(block, comment, source, r, w, x);
+			adjustFragment(program, block.getStart(), name);
 			return block;
 		}
 		catch (LockException e) {
@@ -86,9 +124,9 @@ public class MemoryBlockUtils {
 	 * @param log a {@link StringBuffer} for appending error messages
 	 * @return the new created block
 	 */
-	public static MemoryBlock createBitMappedBlock(Program program, String name,
+	public static MemoryBlock createBitMappedBlock(Program program, String name, Address start,
-			Address start, Address base, int length, String comment, String source, boolean r,
+			Address base, int length, String comment, String source, boolean r, boolean w,
-			boolean w, boolean x, MessageLog log) {
+			boolean x, MessageLog log) {
 
 		Memory memory = program.getMemory();
 		try {
@@ -197,7 +235,7 @@ public class MemoryBlockUtils {
 		}
 
 		setBlockAttributes(block, comment, source, r, w, x);
-		adjustFragment(program, start, name);
+		adjustFragment(program, block.getStart(), name);
 		return block;
 	}
 
@@ -252,8 +290,7 @@ public class MemoryBlockUtils {
 	}
 
 	private static void setBlockAttributes(MemoryBlock block, String comment, String source,
-			boolean r,
+			boolean r, boolean w, boolean x) {
-			boolean w, boolean x) {
 		block.setComment(comment);
 		block.setSourceName(source);
 		block.setRead(r);

Ghidra/Features/Base/src/main/java/ghidra/app/util/bin/format/elf/ElfLoadHelper.java

@@ -15,7 +15,6 @@
  */
 package ghidra.app.util.bin.format.elf;
 
-import ghidra.app.util.MemoryBlockUtil;
 import ghidra.app.util.bin.format.MemoryLoadable;
 import ghidra.app.util.importer.MessageLog;
 import ghidra.program.model.address.Address;
@@ -44,12 +43,6 @@ public interface ElfLoadHelper {
 	 */
 	ElfHeader getElfHeader();
 
-	/**
-	 * Memory block utility (program memory may be accessed directly if preferred)
-	 * @return memory block utility associated with program load
-	 */
-	MemoryBlockUtil getMemoryBlockUtil();
-
 	/**
 	 * Get the message log
 	 * @return message log

Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/ElfLoader.java

@@ -28,12 +28,10 @@ import ghidra.app.util.bin.format.elf.ElfHeader;
 import ghidra.app.util.importer.*;
 import ghidra.framework.model.DomainFolder;
 import ghidra.framework.model.DomainObject;
-import ghidra.program.model.address.AddressSpace;
 import ghidra.program.model.lang.Endian;
 import ghidra.program.model.listing.Program;
 import ghidra.program.util.ELFExternalSymbolResolver;
 import ghidra.util.Msg;
-import ghidra.util.StringUtilities;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.task.TaskMonitor;
 
@@ -76,22 +74,6 @@ public class ElfLoader extends AbstractLibrarySupportLoader {
 		return options;
 	}
 
-	private String getBaseOffsetString(long imageBase, AddressSpace defaultSpace) {
-		int maxNibbles = defaultSpace.getSize() / 4;
-		int minNibbles = Math.min(8, maxNibbles);
-		String baseOffsetStr = Long.toHexString(imageBase);
-		int baseOffsetStrLen = baseOffsetStr.length();
-		if (imageBase < 0) {
-			if (baseOffsetStrLen > maxNibbles) {
-				baseOffsetStr = baseOffsetStr.substring(baseOffsetStrLen - maxNibbles);
-			}
-		}
-		else if (baseOffsetStrLen < minNibbles) {
-			baseOffsetStr = StringUtilities.pad(baseOffsetStr, '0', minNibbles - baseOffsetStrLen);
-		}
-		return baseOffsetStr;
-	}
-
 	@Override
 	public String validateOptions(ByteProvider provider, LoadSpec loadSpec, List<Option> options) {
 		if (options != null) {
@@ -152,7 +134,7 @@ public class ElfLoader extends AbstractLibrarySupportLoader {
 		try {
 			GenericFactory factory = MessageLogContinuesFactory.create(log);
 			ElfHeader elf = ElfHeader.createElfHeader(factory, provider);
-			ElfProgramBuilder.loadElf(elf, program, options, log, handler, monitor);
+			ElfProgramBuilder.loadElf(elf, program, options, log, monitor);
 		}
 		catch (ElfException e) {
 			throw new IOException(e.getMessage());

Ghidra/Features/Base/src/main/java/ghidra/app/util/opinion/ElfProgramBuilder.java

@@ -24,18 +24,19 @@ import java.util.List;
 import java.util.function.Consumer;
 
 import ghidra.app.cmd.label.SetLabelPrimaryCmd;
-import ghidra.app.util.MemoryBlockUtil;
+import ghidra.app.util.MemoryBlockUtils;
 import ghidra.app.util.Option;
+import ghidra.app.util.bin.ByteProvider;
 import ghidra.app.util.bin.format.MemoryLoadable;
 import ghidra.app.util.bin.format.elf.*;
 import ghidra.app.util.bin.format.elf.ElfDynamicType.ElfDynamicValueType;
 import ghidra.app.util.bin.format.elf.extend.ElfLoadAdapter;
 import ghidra.app.util.bin.format.elf.relocation.ElfRelocationContext;
 import ghidra.app.util.bin.format.elf.relocation.ElfRelocationHandler;
-import ghidra.app.util.importer.MemoryConflictHandler;
 import ghidra.app.util.importer.MessageLog;
 import ghidra.framework.options.Options;
 import ghidra.framework.store.LockException;
+import ghidra.program.database.mem.FileBytes;
 import ghidra.program.database.register.AddressRangeObjectMap;
 import ghidra.program.model.address.*;
 import ghidra.program.model.data.*;
@@ -71,7 +72,7 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 	private MessageLog log;
 
 	private ElfHeader elf;
-	private MemoryBlockUtil mbu;
+	private FileBytes fileBytes;
 
 	private Listing listing;
 	private Memory memory;
@@ -79,14 +80,13 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 	private HashMap<ElfSymbol, Address> symbolMap = new HashMap<>();
 
 	protected ElfProgramBuilder(ElfHeader elf, Program program, List<Option> options,
-			MessageLog log, MemoryConflictHandler handler) {
+			MessageLog log) {
 		super(program);
 		this.elf = elf;
 		this.options = options;
 		this.log = log;
 		memory = program.getMemory();
 		listing = program.getListing();
-		mbu = new MemoryBlockUtil(program, handler);
 	}
 
 	@Override
@@ -94,16 +94,9 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 		return elf;
 	}
 
-	@Override
-	public MemoryBlockUtil getMemoryBlockUtil() {
-		return mbu;
-	}
-
 	static void loadElf(ElfHeader elf, Program program, List<Option> options, MessageLog log,
-			MemoryConflictHandler handler, TaskMonitor monitor)
+			TaskMonitor monitor) throws IOException, CancelledException {
-			throws IOException, CancelledException {
+		ElfProgramBuilder elfProgramBuilder = new ElfProgramBuilder(elf, program, options, log);
-		ElfProgramBuilder elfProgramBuilder =
-			new ElfProgramBuilder(elf, program, options, log, handler);
 		elfProgramBuilder.load(monitor);
 	}
 
@@ -128,6 +121,11 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 			processSectionHeaders(monitor);
 
 			// resolve segment/sections and create program memory blocks
+			ByteProvider byteProvider = elf.getReader().getByteProvider();
+			try (InputStream fileIn = byteProvider.getInputStream(0)) {
+				fileBytes = program.getMemory().createFileBytes(byteProvider.getName(), 0,
+					byteProvider.length(), fileIn);
+			}
 			resolve(monitor);
 
 			if (elf.e_shnum() == 0) {
@@ -175,13 +173,6 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 			success = true;
 		}
 		finally {
-			if (mbu != null) {
-				if (log != null) {
-					log(mbu.getMessages());
-				}
-				mbu.dispose();
-				mbu = null;
-			}
 			program.endTransaction(id, success);
 		}
 	}
@@ -2898,6 +2889,9 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 			boolean w, boolean x, TaskMonitor monitor)
 			throws IOException, AddressOverflowException, CancelledException {
 
+		// TODO: MemoryBlockUtil poorly and inconsistently handles duplicate name errors (can throw RuntimeException).
+		// Are we immune from such errors? If not, how should they be handled?
+
 		long revisedLength = checkBlockLimit(name, dataLength, true);
 		long sizeGB = revisedLength >> Memory.GBYTE_SHIFT_FACTOR;
 
@@ -2910,26 +2904,13 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 		Msg.debug(this,
 			"Loading block " + name + " at " + start + " from file offset " + fileOffset);
 
-		try (InputStream dataInput =
-			getInitializedBlockInputStream(loadable, start, fileOffset, revisedLength)) {
-
 		String blockComment = comment;
 		if (dataLength != revisedLength) {
 			blockComment += " (section truncated to " + sizeGB + " GByte)";
 		}
 
-			if (isOverlay) {
+		return MemoryBlockUtils.createInitializedBlock(program, isOverlay, name, start, fileBytes,
-				return mbu.createOverlayBlock(name, start, dataInput, revisedLength, blockComment,
+			fileOffset, revisedLength, blockComment, BLOCK_SOURCE_NAME, r, w, x, log);
-					BLOCK_SOURCE_NAME, r, w, x, monitor);
-			}
-			return mbu.createInitializedBlock(name, start, dataInput, revisedLength, blockComment,
-				BLOCK_SOURCE_NAME, r, w, x, monitor);
-		}
-		catch (DuplicateNameException e) {
-			// TODO: MemoryBlockUtil poorly and inconsistently handles duplicate name errors (can throw RuntimeException).
-			// Are we immune from such errors? If not, how should they be handled?
-			throw new IOException(e);
-		}
 	}
 
 	@Override
@@ -2953,8 +2934,8 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
 			comment += " (section truncated to " + sizeGB + " GByte)";
 		}
 
-		return mbu.createUninitializedBlock(isOverlay, name, start, dataLength, comment,
+		return MemoryBlockUtils.createUninitializedBlock(program, isOverlay, name, start,
-			BLOCK_SOURCE_NAME, r, w, x);
+			revisedLength, comment, BLOCK_SOURCE_NAME, r, w, x, log);
 	}
 
 }

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/database/mem/MemoryMapDBAdapter.java

@@ -163,7 +163,7 @@ abstract class MemoryMapDBAdapter {
 	 * block data will be initialized to zero (0x00).
 	 * @param name the name of the block
 	 * @param startAddr the start address of the block.
-	 * @param is data source
+	 * @param is data source or null for zero initialization
 	 * @param length size of block
 	 * @param permissions the new block permissions
 	 * @return new memory block

Ghidra/Framework/SoftwareModeling/src/main/java/ghidra/program/model/mem/Memory.java

@@ -110,7 +110,7 @@ public interface Memory extends AddressSetView {
 	 * Create an initialized memory block and add it to this Memory.
 	 * @param name block name
 	 * @param start start address of the block
-	 * @param is source of the data used to fill the block.
+	 * @param is source of the data used to fill the block or null for zero initialization.
 	 * @param length the size of the block
 	 * @param monitor task monitor
 	 * @param overlay if true, the block will be created as an OVERLAY which means that a new

Ghidra/Processors/Atmel/src/main/java/ghidra/app/util/bin/format/elf/relocation/AVR32_ElfRelocationHandler.java

@@ -15,16 +15,17 @@
  */
 package ghidra.app.util.bin.format.elf.relocation;
 
+import ghidra.app.util.MemoryBlockUtils;
 import ghidra.app.util.bin.StructConverter;
 import ghidra.app.util.bin.format.elf.*;
-import ghidra.program.model.address.*;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.address.AddressSpace;
 import ghidra.program.model.listing.Listing;
 import ghidra.program.model.listing.Program;
 import ghidra.program.model.mem.Memory;
 import ghidra.program.model.mem.MemoryAccessException;
 import ghidra.program.model.util.CodeUnitInsertionException;
 import ghidra.util.exception.NotFoundException;
-import ghidra.util.task.TaskMonitor;
 
 public class AVR32_ElfRelocationHandler extends ElfRelocationHandler {
 
@@ -144,18 +145,10 @@ public class AVR32_ElfRelocationHandler extends ElfRelocationHandler {
 								newSectionTypeBuff.append("Global External Function");
 							}
 							ElfLoadHelper loadHelper = elfRelocationContext.getLoadHelper();
-							try {
+							MemoryBlockUtils.createInitializedBlock(program, false,
-								loadHelper.getMemoryBlockUtil().createInitializedBlock(
+								newSectionNameBuff.toString(), currNewAddress, currElfSymbolSize,
-									newSectionNameBuff.toString(), currNewAddress, null,
+								newSectionTypeBuff.toString(), "AVR32-ELF Loader", isReadable,
-									currElfSymbolSize, newSectionTypeBuff.toString(), null,
+								isWritable, isExecutable, loadHelper.getLog());
-									isReadable, // Readable
-									isWritable, // Not Writable
-									isExecutable, // Executable
-									TaskMonitor.DUMMY);
-							}
-							catch (AddressOverflowException e) {
-								loadHelper.log(e.getMessage());
-							}
 						}
 					}
 					try {

Ghidra/Processors/MIPS/src/main/java/ghidra/app/util/bin/format/elf/relocation/MIPS_ElfRelocationHandler.java

@@ -17,7 +17,7 @@ package ghidra.app.util.bin.format.elf.relocation;
 
 import java.util.*;
 
-import ghidra.app.util.MemoryBlockUtil;
+import ghidra.app.util.MemoryBlockUtils;
 import ghidra.app.util.bin.format.elf.*;
 import ghidra.app.util.bin.format.elf.extend.MIPS_ElfExtension;
 import ghidra.app.util.importer.MessageLog;
@@ -30,7 +30,6 @@ import ghidra.program.model.symbol.SymbolUtilities;
 import ghidra.util.*;
 import ghidra.util.exception.AssertException;
 import ghidra.util.exception.NotFoundException;
-import ghidra.util.task.TaskMonitor;
 
 public class MIPS_ElfRelocationHandler extends ElfRelocationHandler {
 
@@ -1152,14 +1151,13 @@ public class MIPS_ElfRelocationHandler extends ElfRelocationHandler {
 			if (lastSectionGotEntryAddress == null) {
 				return;
 			}
-			MemoryBlockUtil mbu = loadHelper.getMemoryBlockUtil();
 			int size = (int) lastSectionGotEntryAddress.subtract(sectionGotAddress) + 1;
 			String sectionName = relocationTable.getSectionToBeRelocated().getNameAsString();
 			String blockName = getSectionGotName();
 			try {
-				MemoryBlock block = mbu.createInitializedBlock(blockName, sectionGotAddress, null,
+				MemoryBlock block = MemoryBlockUtils.createInitializedBlock(program, false,
-					size, "GOT for " + sectionName + " section", "MIPS-Elf Loader", true, false,
+					blockName, sectionGotAddress, size, "GOT for " + sectionName + " section",
-					false, TaskMonitor.DUMMY);
+					"MIPS-Elf Loader", true, false, false, loadHelper.getLog());
 				DataConverter converter =
 					program.getMemory().isBigEndian() ? BigEndianDataConverter.INSTANCE
 							: LittleEndianDataConverter.INSTANCE;
@@ -1176,7 +1174,7 @@ public class MIPS_ElfRelocationHandler extends ElfRelocationHandler {
 					loadHelper.createData(addr, PointerDataType.dataType);
 				}
 			}
-			catch (AddressOverflowException | MemoryAccessException e) {
+			catch (MemoryAccessException e) {
 				throw new AssertException(e); // unexpected
 			}
 		}