GT-3064 fixing importing to be cancellable

This commit is contained in:
ghidravore 2019-08-07 14:39:27 -04:00
parent 70757b658e
commit 33c5feac44
22 changed files with 279 additions and 252 deletions

View file

@ -108,8 +108,6 @@ public class MemoryBlockUtils {
return null; return null;
} }
/** /**
* Creates a new bit mapped memory block. (A bit mapped block is a block where each byte value * Creates a new bit mapped memory block. (A bit mapped block is a block where each byte value
* is either 1 or 0 and the value is taken from a bit in a byte at some other address in memory) * is either 1 or 0 and the value is taken from a bit in a byte at some other address in memory)
@ -279,12 +277,12 @@ public class MemoryBlockUtils {
MemoryBlock block; MemoryBlock block;
try { try {
try { try {
block = memory.createInitializedBlock(name, start, dataInput, dataLength, block = memory.createInitializedBlock(name, start, dataInput, dataLength, monitor,
monitor, isOverlay); isOverlay);
} }
catch (MemoryConflictException e) { catch (MemoryConflictException e) {
block = memory.createInitializedBlock(name, start, dataInput, dataLength, block = memory.createInitializedBlock(name, start, dataInput, dataLength, monitor,
monitor, true); true);
} }
} }
catch (LockException | DuplicateNameException | MemoryConflictException e) { catch (LockException | DuplicateNameException | MemoryConflictException e) {
@ -325,11 +323,12 @@ public class MemoryBlockUtils {
* @param program the program in which to create a new FileBytes object * @param program the program in which to create a new FileBytes object
* @param provider the ByteProvider from which to get the bytes. * @param provider the ByteProvider from which to get the bytes.
* @return the newly created FileBytes object. * @return the newly created FileBytes object.
* @param monitor the monitor for canceling this potentially long running operation.
* @throws IOException if an IOException occurred. * @throws IOException if an IOException occurred.
*/ */
public static FileBytes createFileBytes(Program program, ByteProvider provider) public static FileBytes createFileBytes(Program program, ByteProvider provider,
throws IOException { TaskMonitor monitor) throws IOException, CancelledException {
return createFileBytes(program, provider, 0, provider.length()); return createFileBytes(program, provider, 0, provider.length(), monitor);
} }
/** /**
@ -338,14 +337,16 @@ public class MemoryBlockUtils {
* @param provider the ByteProvider from which to get the bytes. * @param provider the ByteProvider from which to get the bytes.
* @param offset the offset into the ByteProvider from which to start loading bytes. * @param offset the offset into the ByteProvider from which to start loading bytes.
* @param length the number of bytes to load * @param length the number of bytes to load
* @param monitor the monitor for canceling this potentially long running operation.
* @return the newly created FileBytes object. * @return the newly created FileBytes object.
* @throws IOException if an IOException occurred. * @throws IOException if an IOException occurred.
* @throws CancelledException if the user cancelled the operation
*/ */
public static FileBytes createFileBytes(Program program, ByteProvider provider, long offset, public static FileBytes createFileBytes(Program program, ByteProvider provider, long offset,
long length) throws IOException { long length, TaskMonitor monitor) throws IOException, CancelledException {
Memory memory = program.getMemory(); Memory memory = program.getMemory();
try (InputStream fis = provider.getInputStream(offset)) { try (InputStream fis = provider.getInputStream(offset)) {
return memory.createFileBytes(provider.getName(), offset, length, fis); return memory.createFileBytes(provider.getName(), offset, length, fis, monitor);
} }
} }

View file

@ -298,7 +298,7 @@ public class BinaryLoader extends AbstractProgramLoader {
@Override @Override
protected boolean loadProgramInto(ByteProvider provider, LoadSpec loadSpec, protected boolean loadProgramInto(ByteProvider provider, LoadSpec loadSpec,
List<Option> options, MessageLog log, Program prog, TaskMonitor monitor) List<Option> options, MessageLog log, Program prog, TaskMonitor monitor)
throws IOException { throws IOException, CancelledException {
long length = getLength(options); long length = getLength(options);
//File file = provider.getFile(); //File file = provider.getFile();
long fileOffset = getFileOffset(options); long fileOffset = getFileOffset(options);
@ -312,7 +312,8 @@ public class BinaryLoader extends AbstractProgramLoader {
length = clipToMemorySpace(length, log, prog); length = clipToMemorySpace(length, log, prog);
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(prog, provider, fileOffset, length); FileBytes fileBytes =
MemoryBlockUtils.createFileBytes(prog, provider, fileOffset, length, monitor);
try { try {
AddressSpace space = prog.getAddressFactory().getDefaultAddressSpace(); AddressSpace space = prog.getAddressFactory().getDefaultAddressSpace();
if (baseAddr == null) { if (baseAddr == null) {

View file

@ -38,8 +38,7 @@ import ghidra.program.model.mem.*;
import ghidra.program.model.symbol.*; import ghidra.program.model.symbol.*;
import ghidra.program.model.util.CodeUnitInsertionException; import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.Msg; import ghidra.util.Msg;
import ghidra.util.exception.InvalidInputException; import ghidra.util.exception.*;
import ghidra.util.exception.NotFoundException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
public class CoffLoader extends AbstractLibrarySupportLoader { public class CoffLoader extends AbstractLibrarySupportLoader {
@ -183,7 +182,8 @@ public class CoffLoader extends AbstractLibrarySupportLoader {
@Override @Override
protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options,
Program program, TaskMonitor monitor, MessageLog log) throws IOException { Program program, TaskMonitor monitor, MessageLog log)
throws IOException, CancelledException {
boolean performFakeLinking = performFakeLinking(options); boolean performFakeLinking = performFakeLinking(options);
@ -193,7 +193,7 @@ public class CoffLoader extends AbstractLibrarySupportLoader {
Map<CoffSectionHeader, Address> sectionsMap = new HashMap<>(); Map<CoffSectionHeader, Address> sectionsMap = new HashMap<>();
Map<CoffSymbol, Symbol> symbolsMap = new HashMap<>(); Map<CoffSymbol, Symbol> symbolsMap = new HashMap<>();
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider); FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, monitor);
int id = program.startTransaction("loading program from COFF"); int id = program.startTransaction("loading program from COFF");
boolean success = false; boolean success = false;

View file

@ -49,7 +49,6 @@ public class DyldCacheLoader extends AbstractLibrarySupportLoader {
/** Default value for loader option to create memory blocks for DYLIB sections */ /** Default value for loader option to create memory blocks for DYLIB sections */
static final boolean CREATE_DYLIB_SECTIONS_OPTION_DEFAULT = false; static final boolean CREATE_DYLIB_SECTIONS_OPTION_DEFAULT = false;
@Override @Override
public Collection<LoadSpec> findSupportedLoadSpecs(ByteProvider provider) throws IOException { public Collection<LoadSpec> findSupportedLoadSpecs(ByteProvider provider) throws IOException {
List<LoadSpec> loadSpecs = new ArrayList<>(); List<LoadSpec> loadSpecs = new ArrayList<>();
@ -84,8 +83,8 @@ public class DyldCacheLoader extends AbstractLibrarySupportLoader {
try { try {
DyldCacheProgramBuilder.buildProgram(program, provider, DyldCacheProgramBuilder.buildProgram(program, provider,
MemoryBlockUtils.createFileBytes(program, provider), shouldProcessSymbols(options), MemoryBlockUtils.createFileBytes(program, provider, monitor),
shouldCreateDylibSections(options), log, monitor); shouldProcessSymbols(options), shouldCreateDylibSections(options), log, monitor);
} }
catch (CancelledException e) { catch (CancelledException e) {
return; return;

View file

@ -120,7 +120,7 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
ByteProvider byteProvider = elf.getReader().getByteProvider(); ByteProvider byteProvider = elf.getReader().getByteProvider();
try (InputStream fileIn = byteProvider.getInputStream(0)) { try (InputStream fileIn = byteProvider.getInputStream(0)) {
fileBytes = program.getMemory().createFileBytes(byteProvider.getName(), 0, fileBytes = program.getMemory().createFileBytes(byteProvider.getName(), 0,
byteProvider.length(), fileIn); byteProvider.length(), fileIn, monitor);
} }
// process headers and define "section" within memory elfProgramBuilder // process headers and define "section" within memory elfProgramBuilder
@ -2984,7 +2984,7 @@ class ElfProgramBuilder extends MemorySectionResolver implements ElfLoadHelper {
// Are we immune from such errors? If not, how should they be handled? // Are we immune from such errors? If not, how should they be handled?
long revisedLength = checkBlockLimit(name, dataLength, false); long revisedLength = checkBlockLimit(name, dataLength, false);
if (start.isNonLoadedMemoryAddress()) { if (start.isNonLoadedMemoryAddress()) {
r = false; r = false;
w = false; w = false;

View file

@ -81,7 +81,7 @@ public class MachoLoader extends AbstractLibrarySupportLoader {
Program program, TaskMonitor monitor, MessageLog log) throws IOException { Program program, TaskMonitor monitor, MessageLog log) throws IOException {
try { try {
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider); FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, monitor);
// A Mach-O file may contain PRELINK information. If so, we use a special // A Mach-O file may contain PRELINK information. If so, we use a special
// program builder that knows how to deal with it. // program builder that knows how to deal with it.
@ -134,7 +134,7 @@ public class MachoLoader extends AbstractLibrarySupportLoader {
log.appendMsg("WARNING! No archives found in the UBI: " + libFile); log.appendMsg("WARNING! No archives found in the UBI: " + libFile);
return false; return false;
} }
for (FatArch architecture : architectures) { for (FatArch architecture : architectures) {
// Note: The creation of the byte provider that we pass to the importer deserves a // Note: The creation of the byte provider that we pass to the importer deserves a

View file

@ -87,9 +87,9 @@ public class MzLoader extends AbstractLibrarySupportLoader {
@Override @Override
public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, Program prog, public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, Program prog,
TaskMonitor monitor, MessageLog log) throws IOException { TaskMonitor monitor, MessageLog log) throws IOException, CancelledException {
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(prog, provider); FileBytes fileBytes = MemoryBlockUtils.createFileBytes(prog, provider, monitor);
AddressFactory af = prog.getAddressFactory(); AddressFactory af = prog.getAddressFactory();
if (!(af.getDefaultAddressSpace() instanceof SegmentedAddressSpace)) { if (!(af.getDefaultAddressSpace() instanceof SegmentedAddressSpace)) {
throw new IOException("Selected Language must have a segmented address space."); throw new IOException("Selected Language must have a segmented address space.");
@ -350,7 +350,7 @@ public class MzLoader extends AbstractLibrarySupportLoader {
try { try {
Memory mem = prog.getMemory(); Memory mem = prog.getMemory();
SegmentedAddressSpace space = SegmentedAddressSpace space =
(SegmentedAddressSpace) prog.getAddressFactory().getDefaultAddressSpace(); (SegmentedAddressSpace) prog.getAddressFactory().getDefaultAddressSpace();
int relocationTableOffset = Conv.shortToInt(dos.e_lfarlc()); int relocationTableOffset = Conv.shortToInt(dos.e_lfarlc());
int csStart = INITIAL_SEGMENT_VAL; int csStart = INITIAL_SEGMENT_VAL;

View file

@ -40,8 +40,7 @@ import ghidra.program.model.symbol.*;
import ghidra.program.model.util.CodeUnitInsertionException; import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.Conv; import ghidra.util.Conv;
import ghidra.util.Msg; import ghidra.util.Msg;
import ghidra.util.exception.DuplicateNameException; import ghidra.util.exception.*;
import ghidra.util.exception.InvalidInputException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
/** /**
@ -84,7 +83,7 @@ public class NeLoader extends AbstractLibrarySupportLoader {
@Override @Override
public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, Program prog, public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, Program prog,
TaskMonitor monitor, MessageLog log) throws IOException { TaskMonitor monitor, MessageLog log) throws IOException, CancelledException {
if (monitor.isCancelled()) { if (monitor.isCancelled()) {
return; return;
@ -98,7 +97,7 @@ public class NeLoader extends AbstractLibrarySupportLoader {
// We don't use the file bytes to create block because the bytes are manipulated before // We don't use the file bytes to create block because the bytes are manipulated before
// forming the block. Creating the FileBytes anyway in case later we want access to all // forming the block. Creating the FileBytes anyway in case later we want access to all
// the original bytes. // the original bytes.
MemoryBlockUtils.createFileBytes(prog, provider); MemoryBlockUtils.createFileBytes(prog, provider, monitor);
NewExecutable ne = new NewExecutable(factory, provider); NewExecutable ne = new NewExecutable(factory, provider);
WindowsHeader wh = ne.getWindowsHeader(); WindowsHeader wh = ne.getWindowsHeader();

View file

@ -34,6 +34,7 @@ import ghidra.program.model.mem.*;
import ghidra.program.model.symbol.*; import ghidra.program.model.symbol.*;
import ghidra.program.model.util.CodeUnitInsertionException; import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.*; import ghidra.util.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.InvalidInputException; import ghidra.util.exception.InvalidInputException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
import ghidra.util.task.TaskMonitorAdapter; import ghidra.util.task.TaskMonitorAdapter;
@ -107,7 +108,8 @@ public class OmfLoader extends AbstractLibrarySupportLoader {
@Override @Override
protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options,
Program program, TaskMonitor monitor, MessageLog log) throws IOException { Program program, TaskMonitor monitor, MessageLog log)
throws IOException, CancelledException {
OmfFileHeader header = null; OmfFileHeader header = null;
BinaryReader reader = OmfFileHeader.createReader(provider); BinaryReader reader = OmfFileHeader.createReader(provider);
@ -127,7 +129,7 @@ public class OmfLoader extends AbstractLibrarySupportLoader {
// We don't use the file bytes to create block because the bytes are manipulated before // We don't use the file bytes to create block because the bytes are manipulated before
// forming the block. Creating the FileBytes anyway in case later we want access to all // forming the block. Creating the FileBytes anyway in case later we want access to all
// the original bytes. // the original bytes.
MemoryBlockUtils.createFileBytes(program, provider); MemoryBlockUtils.createFileBytes(program, provider, monitor);
int id = program.startTransaction("loading program from OMF"); int id = program.startTransaction("loading program from OMF");
boolean success = false; boolean success = false;

View file

@ -44,8 +44,7 @@ import ghidra.program.model.symbol.*;
import ghidra.program.model.util.AddressSetPropertyMap; import ghidra.program.model.util.AddressSetPropertyMap;
import ghidra.program.model.util.CodeUnitInsertionException; import ghidra.program.model.util.CodeUnitInsertionException;
import ghidra.util.*; import ghidra.util.*;
import ghidra.util.exception.DuplicateNameException; import ghidra.util.exception.*;
import ghidra.util.exception.InvalidInputException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
/** /**
@ -94,7 +93,8 @@ public class PeLoader extends AbstractPeDebugLoader {
@Override @Override
protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, protected void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options,
Program program, TaskMonitor monitor, MessageLog log) throws IOException { Program program, TaskMonitor monitor, MessageLog log)
throws IOException, CancelledException {
if (monitor.isCancelled()) { if (monitor.isCancelled()) {
return; return;
@ -112,7 +112,7 @@ public class PeLoader extends AbstractPeDebugLoader {
FileHeader fileHeader = ntHeader.getFileHeader(); FileHeader fileHeader = ntHeader.getFileHeader();
monitor.setMessage("Completing PE header parsing..."); monitor.setMessage("Completing PE header parsing...");
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider); FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, monitor);
try { try {
Map<Integer, Address> sectionNumberToAddress = Map<Integer, Address> sectionNumberToAddress =
processMemoryBlocks(pe, program, fileBytes, monitor, log); processMemoryBlocks(pe, program, fileBytes, monitor, log);

View file

@ -33,6 +33,7 @@ import ghidra.program.model.listing.Data;
import ghidra.program.model.listing.Program; import ghidra.program.model.listing.Program;
import ghidra.program.model.mem.MemoryBlock; import ghidra.program.model.mem.MemoryBlock;
import ghidra.program.model.symbol.*; import ghidra.program.model.symbol.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
public class PefLoader extends AbstractLibrarySupportLoader { public class PefLoader extends AbstractLibrarySupportLoader {
@ -68,9 +69,10 @@ public class PefLoader extends AbstractLibrarySupportLoader {
@Override @Override
public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options, public void load(ByteProvider provider, LoadSpec loadSpec, List<Option> options,
Program program, TaskMonitor monitor, MessageLog log) throws IOException { Program program, TaskMonitor monitor, MessageLog log)
throws IOException, CancelledException {
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider); FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, monitor);
ImportStateCache importState = null; ImportStateCache importState = null;
try { try {
@ -154,8 +156,8 @@ public class PefLoader extends AbstractLibrarySupportLoader {
} }
if (mainSection.getSectionKind() == SectionKind.PackedData || if (mainSection.getSectionKind() == SectionKind.PackedData ||
mainSection.getSectionKind() == SectionKind.UnpackedData || mainSection.getSectionKind() == SectionKind.UnpackedData ||
mainSection.getSectionKind() == SectionKind.ExecutableData) { mainSection.getSectionKind() == SectionKind.ExecutableData) {
CreateDataCmd cmd = new CreateDataCmd(mainAddress, new PointerDataType()); CreateDataCmd cmd = new CreateDataCmd(mainAddress, new PointerDataType());
cmd.applyTo(program); cmd.applyTo(program);
@ -258,12 +260,13 @@ public class PefLoader extends AbstractLibrarySupportLoader {
} }
if (symbolIndex % 100 == 0) { if (symbolIndex % 100 == 0) {
monitor.setMessage("Processing import " + symbolIndex + " of " + symbols.size()); monitor.setMessage(
"Processing import " + symbolIndex + " of " + symbols.size());
} }
++symbolIndex; ++symbolIndex;
String symbolName = String symbolName =
SymbolUtilities.replaceInvalidChars(symbols.get(i).getName(), true); SymbolUtilities.replaceInvalidChars(symbols.get(i).getName(), true);
boolean success = importState.createLibrarySymbol(library, symbolName, start); boolean success = importState.createLibrarySymbol(library, symbolName, start);
if (!success) { if (!success) {
@ -291,8 +294,8 @@ public class PefLoader extends AbstractLibrarySupportLoader {
private void addExternalReference(Program program, Address start, String libraryName, private void addExternalReference(Program program, Address start, String libraryName,
String symbolName, MessageLog log) { String symbolName, MessageLog log) {
try { try {
program.getReferenceManager().addExternalReference(start, libraryName, symbolName, program.getReferenceManager().addExternalReference(start, libraryName, symbolName, null,
null, SourceType.IMPORTED, 0, RefType.DATA); SourceType.IMPORTED, 0, RefType.DATA);
} }
catch (Exception e) { catch (Exception e) {
log.appendMsg(e.getMessage()); log.appendMsg(e.getMessage());
@ -324,7 +327,7 @@ public class PefLoader extends AbstractLibrarySupportLoader {
return; return;
} }
RelocationState state = RelocationState state =
new RelocationState(header, relocationHeader, program, importState); new RelocationState(header, relocationHeader, program, importState);
List<Relocation> relocations = relocationHeader.getRelocations(); List<Relocation> relocations = relocationHeader.getRelocations();
int relocationIndex = 0; int relocationIndex = 0;
for (Relocation relocation : relocations) { for (Relocation relocation : relocations) {
@ -332,8 +335,8 @@ public class PefLoader extends AbstractLibrarySupportLoader {
return; return;
} }
if (relocationIndex % 100 == 0) { if (relocationIndex % 100 == 0) {
monitor.setMessage("Processing relocation " + relocationIndex + " of " + monitor.setMessage(
relocations.size()); "Processing relocation " + relocationIndex + " of " + relocations.size());
} }
++relocationIndex; ++relocationIndex;
@ -362,7 +365,7 @@ public class PefLoader extends AbstractLibrarySupportLoader {
MemoryBlock block = importState.getMemoryBlockForSection(section); MemoryBlock block = importState.getMemoryBlockForSection(section);
Address symbolAddr = block.getStart().add(symbol.getSymbolValue()); Address symbolAddr = block.getStart().add(symbol.getSymbolValue());
AddUniqueLabelCmd cmd = AddUniqueLabelCmd cmd =
new AddUniqueLabelCmd(symbolAddr, symbol.getName(), null, SourceType.IMPORTED); new AddUniqueLabelCmd(symbolAddr, symbol.getName(), null, SourceType.IMPORTED);
if (!cmd.applyTo(program)) { if (!cmd.applyTo(program)) {
log.appendMsg(cmd.getStatusMsg()); log.appendMsg(cmd.getStatusMsg());
} }
@ -372,7 +375,7 @@ public class PefLoader extends AbstractLibrarySupportLoader {
private void processSections(ContainerHeader header, Program program, FileBytes fileBytes, private void processSections(ContainerHeader header, Program program, FileBytes fileBytes,
ImportStateCache importState, MessageLog log, TaskMonitor monitor) ImportStateCache importState, MessageLog log, TaskMonitor monitor)
throws AddressOverflowException, IOException { throws AddressOverflowException, IOException {
List<SectionHeader> sections = header.getSections(); List<SectionHeader> sections = header.getSections();
for (SectionHeader section : sections) { for (SectionHeader section : sections) {

View file

@ -114,8 +114,8 @@ class MyTestMemory extends AddressSet implements Memory {
} }
@Override @Override
public FileBytes createFileBytes(String filename, long offset, long size, InputStream is) public FileBytes createFileBytes(String filename, long offset, long size, InputStream is,
throws IOException { TaskMonitor monitor) throws IOException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

View file

@ -187,7 +187,7 @@ public class PrelinkFileSystem extends GFileSystemBase implements GFileSystemPro
boolean success = false; boolean success = false;
try { try {
FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, offset, FileBytes fileBytes = MemoryBlockUtils.createFileBytes(program, provider, offset,
provider.length() - offset); provider.length() - offset, monitor);
ByteProvider providerWrapper = ByteProvider providerWrapper =
new ByteProviderWrapper(provider, offset, provider.length() - offset); new ByteProviderWrapper(provider, offset, provider.length() - offset);
MachoProgramBuilder.buildProgram(program, providerWrapper, fileBytes, new MessageLog(), MachoProgramBuilder.buildProgram(program, providerWrapper, fileBytes, new MessageLog(),

View file

@ -328,8 +328,8 @@ public class MemoryTestDummy extends AddressSet implements Memory {
} }
@Override @Override
public FileBytes createFileBytes(String filename, long offset, long size, InputStream is) public FileBytes createFileBytes(String filename, long offset, long size, InputStream is,
throws IOException { TaskMonitor monitor) throws IOException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

View file

@ -1,6 +1,5 @@
/* ### /* ###
* IP: GHIDRA * IP: GHIDRA
* REVIEWED: YES
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -16,231 +15,234 @@
*/ */
package ghidra.util; package ghidra.util;
import java.io.IOException;
import java.io.InputStream;
import ghidra.util.exception.IOCancelledException; import ghidra.util.exception.IOCancelledException;
import ghidra.util.task.TaskMonitor; import ghidra.util.task.TaskMonitor;
import java.io.*;
/** /**
* An InputStream which utilizes a TaskMonitor to indicate input progress and * An InputStream which utilizes a TaskMonitor to indicate input progress and
* allows the operation to be cancelled via the TaskMonitor. * allows the operation to be cancelled via the TaskMonitor.
*/ */
public class MonitoredInputStream extends InputStream { public class MonitoredInputStream extends InputStream {
private final static int PROGRESS_INCREMENT = 32*1024; private final static int PROGRESS_INCREMENT = 32 * 1024;
protected InputStream in; protected InputStream in;
private TaskMonitor monitor; private TaskMonitor monitor;
private int smallCount = 0; private int smallCount = 0;
private int count = 0; private int count = 0;
public MonitoredInputStream(InputStream in, TaskMonitor monitor) { public MonitoredInputStream(InputStream in, TaskMonitor monitor) {
this.in = in; this.in = in;
this.monitor = monitor; this.monitor = monitor;
} }
/** /**
* Reset the current progress count to the specified value. * Reset the current progress count to the specified value.
*/ */
public void setProgress(int count) { public void setProgress(int count) {
this.count = count; this.count = count;
} }
/** /**
* Reads the next byte of data from this input stream. The value * Reads the next byte of data from this input stream. The value
* byte is returned as an <code>int</code> in the range * byte is returned as an <code>int</code> in the range
* <code>0</code> to <code>255</code>. If no byte is available * <code>0</code> to <code>255</code>. If no byte is available
* because the end of the stream has been reached, the value * because the end of the stream has been reached, the value
* <code>-1</code> is returned. This method blocks until input data * <code>-1</code> is returned. This method blocks until input data
* is available, the end of the stream is detected, or an exception * is available, the end of the stream is detected, or an exception
* is thrown. * is thrown.
* <p> * <p>
* This method * This method
* simply performs <code>in.read()</code> and returns the result. * simply performs <code>in.read()</code> and returns the result.
* *
* @return the next byte of data, or <code>-1</code> if the end of the * @return the next byte of data, or <code>-1</code> if the end of the
* stream is reached. * stream is reached.
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
*/ */
@Override @Override
public int read() throws IOException { public int read() throws IOException {
if (monitor.isCancelled()) {
throw new IOCancelledException();
}
int n = in.read(); int n = in.read();
if (n != -1) { if (n != -1) {
++smallCount; ++smallCount;
if (smallCount >= PROGRESS_INCREMENT) { if (smallCount >= PROGRESS_INCREMENT) {
if (monitor.isCancelled())
throw new IOCancelledException();
count += smallCount; count += smallCount;
smallCount = 0; smallCount = 0;
monitor.setProgress(count); monitor.setProgress(count);
} }
} }
return n; return n;
} }
/** /**
* Reads up to <code>byte.length</code> bytes of data from this * Reads up to <code>byte.length</code> bytes of data from this
* input stream into an array of bytes. This method blocks until some * input stream into an array of bytes. This method blocks until some
* input is available. * input is available.
* <p> * <p>
* This method simply performs the call * This method simply performs the call
* <code>read(b, 0, b.length)</code> and returns * <code>read(b, 0, b.length)</code> and returns
* the result. It is important that it does * the result. It is important that it does
* <i>not</i> do <code>in.read(b)</code> instead; * <i>not</i> do <code>in.read(b)</code> instead;
* certain subclasses of <code>FilterInputStream</code> * certain subclasses of <code>FilterInputStream</code>
* depend on the implementation strategy actually * depend on the implementation strategy actually
* used. * used.
* *
* @param b the buffer into which the data is read. * @param b the buffer into which the data is read.
* @return the total number of bytes read into the buffer, or * @return the total number of bytes read into the buffer, or
* <code>-1</code> if there is no more data because the end of * <code>-1</code> if there is no more data because the end of
* the stream has been reached. * the stream has been reached.
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
* @see java.io.FilterInputStream#read(byte[], int, int) * @see java.io.FilterInputStream#read(byte[], int, int)
*/ */
@Override @Override
public int read(byte b[]) throws IOException { public int read(byte b[]) throws IOException {
return read(b, 0, b.length); return read(b, 0, b.length);
} }
/** /**
* Reads up to <code>len</code> bytes of data from this input stream * Reads up to <code>len</code> bytes of data from this input stream
* into an array of bytes. This method blocks until some input is * into an array of bytes. This method blocks until some input is
* available. * available.
* <p> * <p>
* This method simply performs <code>in.read(b, off, len)</code> * This method simply performs <code>in.read(b, off, len)</code>
* and returns the result. * and returns the result.
* *
* @param b the buffer into which the data is read. * @param b the buffer into which the data is read.
* @param off the start offset of the data. * @param off the start offset of the data.
* @param len the maximum number of bytes read. * @param len the maximum number of bytes read.
* @return the total number of bytes read into the buffer, or * @return the total number of bytes read into the buffer, or
* <code>-1</code> if there is no more data because the end of * <code>-1</code> if there is no more data because the end of
* the stream has been reached. * the stream has been reached.
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
*/ */
@Override @Override
public int read(byte b[], int off, int len) throws IOException { public int read(byte b[], int off, int len) throws IOException {
if (monitor.isCancelled()) {
throw new IOCancelledException();
}
int n = in.read(b, off, len); int n = in.read(b, off, len);
smallCount += n; smallCount += n;
if (smallCount >= PROGRESS_INCREMENT) { if (smallCount >= PROGRESS_INCREMENT) {
if (monitor.isCancelled())
throw new IOCancelledException();
count += smallCount; count += smallCount;
smallCount = 0; smallCount = 0;
monitor.setProgress(count); monitor.setProgress(count);
} }
return n; return n;
} }
/** /**
* Skips over and discards <code>n</code> bytes of data from the * Skips over and discards <code>n</code> bytes of data from the
* input stream. The <code>skip</code> method may, for a variety of * input stream. The <code>skip</code> method may, for a variety of
* reasons, end up skipping over some smaller number of bytes, * reasons, end up skipping over some smaller number of bytes,
* possibly <code>0</code>. The actual number of bytes skipped is * possibly <code>0</code>. The actual number of bytes skipped is
* returned. * returned.
* <p> * <p>
* This method * This method
* simply performs <code>in.skip(n)</code>. * simply performs <code>in.skip(n)</code>.
* *
* @param n the number of bytes to be skipped. * @param n the number of bytes to be skipped.
* @return the actual number of bytes skipped. * @return the actual number of bytes skipped.
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
*/ */
@Override @Override
public long skip(long n) throws IOException { public long skip(long n) throws IOException {
return in.skip(n); return in.skip(n);
} }
/** /**
* Returns the number of bytes that can be read from this input * Returns the number of bytes that can be read from this input
* stream without blocking. * stream without blocking.
* <p> * <p>
* This method * This method
* simply performs <code>in.available()</code> and * simply performs <code>in.available()</code> and
* returns the result. * returns the result.
* *
* @return the number of bytes that can be read from the input stream * @return the number of bytes that can be read from the input stream
* without blocking. * without blocking.
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
*/ */
@Override @Override
public int available() throws IOException { public int available() throws IOException {
return in.available(); return in.available();
} }
/** /**
* Closes this input stream and releases any system resources * Closes this input stream and releases any system resources
* associated with the stream. * associated with the stream.
* This * This
* method simply performs <code>in.close()</code>. * method simply performs <code>in.close()</code>.
* *
* @exception IOException if an I/O error occurs. * @exception IOException if an I/O error occurs.
*/ */
@Override @Override
public void close() throws IOException { public void close() throws IOException {
in.close(); in.close();
} }
/** /**
* Marks the current position in this input stream. A subsequent * Marks the current position in this input stream. A subsequent
* call to the <code>reset</code> method repositions this stream at * call to the <code>reset</code> method repositions this stream at
* the last marked position so that subsequent reads re-read the same bytes. * the last marked position so that subsequent reads re-read the same bytes.
* <p> * <p>
* The <code>readlimit</code> argument tells this input stream to * The <code>readlimit</code> argument tells this input stream to
* allow that many bytes to be read before the mark position gets * allow that many bytes to be read before the mark position gets
* invalidated. * invalidated.
* <p> * <p>
* This method simply performs <code>in.mark(readlimit)</code>. * This method simply performs <code>in.mark(readlimit)</code>.
* *
* @param readlimit the maximum limit of bytes that can be read before * @param readlimit the maximum limit of bytes that can be read before
* the mark position becomes invalid. * the mark position becomes invalid.
* @see java.io.FilterInputStream#reset * @see java.io.FilterInputStream#reset
*/ */
@Override @Override
public synchronized void mark(int readlimit) { public synchronized void mark(int readlimit) {
in.mark(readlimit); in.mark(readlimit);
} }
/** /**
* Repositions this stream to the position at the time the * Repositions this stream to the position at the time the
* <code>mark</code> method was last called on this input stream. * <code>mark</code> method was last called on this input stream.
* <p> * <p>
* This method * This method
* simply performs <code>in.reset()</code>. * simply performs <code>in.reset()</code>.
* <p> * <p>
* Stream marks are intended to be used in * Stream marks are intended to be used in
* situations where you need to read ahead a little to see what's in * situations where you need to read ahead a little to see what's in
* the stream. Often this is most easily done by invoking some * the stream. Often this is most easily done by invoking some
* general parser. If the stream is of the type handled by the * general parser. If the stream is of the type handled by the
* parse, it just chugs along happily. If the stream is not of * parse, it just chugs along happily. If the stream is not of
* that type, the parser should toss an exception when it fails. * that type, the parser should toss an exception when it fails.
* If this happens within readlimit bytes, it allows the outer * If this happens within readlimit bytes, it allows the outer
* code to reset the stream and try another parser. * code to reset the stream and try another parser.
* *
* @exception IOException if the stream has not been marked or if the * @exception IOException if the stream has not been marked or if the
* mark has been invalidated. * mark has been invalidated.
* @see java.io.FilterInputStream#mark(int) * @see java.io.FilterInputStream#mark(int)
*/ */
@Override @Override
public synchronized void reset() throws IOException { public synchronized void reset() throws IOException {
in.reset(); in.reset();
} }
/** /**
* Tests if this input stream supports the <code>mark</code> * Tests if this input stream supports the <code>mark</code>
* and <code>reset</code> methods. * and <code>reset</code> methods.
* This method * This method
* simply performs <code>in.markSupported()</code>. * simply performs <code>in.markSupported()</code>.
* *
* @return <code>true</code> if this stream type supports the * @return <code>true</code> if this stream type supports the
* <code>mark</code> and <code>reset</code> method; * <code>mark</code> and <code>reset</code> method;
* <code>false</code> otherwise. * <code>false</code> otherwise.
* @see java.io.InputStream#mark(int) * @see java.io.InputStream#mark(int)
* @see java.io.InputStream#reset() * @see java.io.InputStream#reset()
*/ */
@Override @Override
public boolean markSupported() { public boolean markSupported() {
return in.markSupported(); return in.markSupported();
} }
} }

View file

@ -20,6 +20,7 @@ import java.io.InputStream;
import java.util.*; import java.util.*;
import db.*; import db.*;
import ghidra.util.exception.IOCancelledException;
import ghidra.util.exception.VersionException; import ghidra.util.exception.VersionException;
/** /**
@ -158,9 +159,18 @@ class FileBytesAdapterV0 extends FileBytesAdapter {
} }
buffers[bufCount - 1] = handle.createBuffer(sizeLastBuf); buffers[bufCount - 1] = handle.createBuffer(sizeLastBuf);
for (DBBuffer buffer : buffers) { try {
buffer.fill(is); for (DBBuffer buffer : buffers) {
buffer.fill(is);
}
}
catch (IOCancelledException e) {
for (DBBuffer buffer : buffers) {
buffer.delete();
}
throw e;
} }
return buffers; return buffers;
} }
} }

View file

@ -494,8 +494,10 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
return newBlock; return newBlock;
} }
catch (IOCancelledException e) { catch (IOCancelledException e) {
// TODO: this could leave things in a bad state. // this assumes the adapter has already cleaned up any partially created buffers.
// Canceling requires additional improvements (see GT-3064) if (overlay) {
checkRemoveAddressSpace(start.getAddressSpace());
}
throw new CancelledException(); throw new CancelledException();
} }
catch (IOException e) { catch (IOException e) {
@ -2022,14 +2024,18 @@ public class MemoryMapDB implements Memory, ManagerDB, LiveMemoryListener {
} }
@Override @Override
public FileBytes createFileBytes(String filename, long offset, long size, InputStream is) public FileBytes createFileBytes(String filename, long offset, long size, InputStream is,
throws IOException { TaskMonitor monitor) throws IOException, CancelledException {
lock.acquire(); lock.acquire();
try { try {
// TODO: this should accept task monitor to permit cancellation although if (monitor != null && is != null) {
// canceling requires additional improvements (see GT-3064) is = new MonitoredInputStream(is, monitor);
}
return fileBytesAdapter.createFileBytes(filename, offset, size, is); return fileBytesAdapter.createFileBytes(filename, offset, size, is);
} }
catch (IOCancelledException e) {
throw new CancelledException();
}
finally { finally {
lock.release(); lock.release();
} }

View file

@ -148,6 +148,8 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter {
@Override @Override
MemoryBlockDB createInitializedBlock(String name, Address startAddr, InputStream is, MemoryBlockDB createInitializedBlock(String name, Address startAddr, InputStream is,
long length, int permissions) throws AddressOverflowException, IOException { long length, int permissions) throws AddressOverflowException, IOException {
// TODO verify that it is necessary to pre-define all segments in the address map
updateAddressMapForAllAddresses(startAddr, length); updateAddressMapForAllAddresses(startAddr, length);
List<SubMemoryBlock> subBlocks = new ArrayList<>(); List<SubMemoryBlock> subBlocks = new ArrayList<>();
@ -171,7 +173,7 @@ public class MemoryMapDBAdapterV3 extends MemoryMapDBAdapter {
Collections.sort(memoryBlocks); Collections.sort(memoryBlocks);
return newBlock; return newBlock;
} }
catch (IOException e) { catch (IOCancelledException e) {
// clean up any created DBBufferss // clean up any created DBBufferss
for (SubMemoryBlock subMemoryBlock : subBlocks) { for (SubMemoryBlock subMemoryBlock : subBlocks) {
BufferSubMemoryBlock bufferSubMemoryBlock = (BufferSubMemoryBlock) subMemoryBlock; BufferSubMemoryBlock bufferSubMemoryBlock = (BufferSubMemoryBlock) subMemoryBlock;

View file

@ -729,11 +729,14 @@ public interface Memory extends AddressSetView {
* @param offset the offset into the file for the first byte in the input stream. * @param offset the offset into the file for the first byte in the input stream.
* @param size the number of bytes to store from the input stream. * @param size the number of bytes to store from the input stream.
* @param is the input stream that will supply the bytes to store in the program. * @param is the input stream that will supply the bytes to store in the program.
* @param monitor
* @return a FileBytes that was created to access the bytes. * @return a FileBytes that was created to access the bytes.
* @throws IOException if there was an IOException saving the bytes to the program database. * @throws IOException if there was an IOException saving the bytes to the program database.
* @throws CancelledException if the user cancelled this operation. Note: the database will
* be stable, but the buffers may contain 0s instead of the actual bytes.
*/ */
public FileBytes createFileBytes(String filename, long offset, long size, InputStream is) public FileBytes createFileBytes(String filename, long offset, long size, InputStream is,
throws IOException; TaskMonitor monitor) throws IOException, CancelledException;
/** /**
* Returns a list of all the stored original file bytes objects * Returns a list of all the stored original file bytes objects

View file

@ -469,8 +469,8 @@ public class MemoryStub implements Memory {
} }
@Override @Override
public FileBytes createFileBytes(String filename, long offset, long size, InputStream is) public FileBytes createFileBytes(String filename, long offset, long size, InputStream is,
throws IOException { TaskMonitor monitor) throws IOException {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }

View file

@ -47,7 +47,7 @@ public class FileBytesTest extends AbstractGenericTest {
} }
@Test @Test
public void testStoreAndRetrieveFileBytes() throws IOException { public void testStoreAndRetrieveFileBytes() throws Exception {
int dataSize = MAX_BUFFER_SIZE_FOR_TESTING / 2; int dataSize = MAX_BUFFER_SIZE_FOR_TESTING / 2;
FileBytes fileBytes = createFileBytes("testFile", dataSize); FileBytes fileBytes = createFileBytes("testFile", dataSize);
@ -186,13 +186,13 @@ public class FileBytesTest extends AbstractGenericTest {
} }
} }
private FileBytes createFileBytes(String name, int size) throws IOException { private FileBytes createFileBytes(String name, int size) throws Exception {
byte[] bytes = new byte[size]; byte[] bytes = new byte[size];
for (int i = 0; i < size; i++) { for (int i = 0; i < size; i++) {
bytes[i] = (byte) i; bytes[i] = (byte) i;
} }
try (ByteArrayInputStream is = new ByteArrayInputStream(bytes)) { try (ByteArrayInputStream is = new ByteArrayInputStream(bytes)) {
return mem.createFileBytes(name, 0, size, is); return mem.createFileBytes(name, 0, size, is, TaskMonitor.DUMMY);
} }
} }

View file

@ -18,7 +18,6 @@ package ghidra.program.database.mem;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List; import java.util.List;
import org.junit.*; import org.junit.*;
@ -282,7 +281,8 @@ public class MemBlockDBTest extends AbstractGenericTest {
@Test @Test
public void testCreateFileBytesBlockOutSideRange() throws Exception { public void testCreateFileBytesBlockOutSideRange() throws Exception {
byte[] bytes = new byte[256]; byte[] bytes = new byte[256];
FileBytes fileBytes = mem.createFileBytes("test", 0, 100, new ByteArrayInputStream(bytes)); FileBytes fileBytes =
mem.createFileBytes("test", 0, 100, new ByteArrayInputStream(bytes), TaskMonitor.DUMMY);
try { try {
mem.createInitializedBlock("test", addr(100), fileBytes, 10, 100, false); mem.createInitializedBlock("test", addr(100), fileBytes, 10, 100, false);
fail( fail(
@ -379,7 +379,6 @@ public class MemBlockDBTest extends AbstractGenericTest {
} }
} }
@Test @Test
public void testJoinFileBytesBlockAndBufferBlock() throws Exception { public void testJoinFileBytesBlockAndBufferBlock() throws Exception {
FileBytes fileBytes = createFileBytes(); FileBytes fileBytes = createFileBytes();
@ -695,7 +694,6 @@ public class MemBlockDBTest extends AbstractGenericTest {
assertEquals(2, ranges.getRangeCount()); // we have two sublocks so two distinct ranges assertEquals(2, ranges.getRangeCount()); // we have two sublocks so two distinct ranges
assertEquals(10, ranges.get(0).getSize() + ranges.get(1).getSize()); assertEquals(10, ranges.get(0).getSize() + ranges.get(1).getSize());
ByteSourceRange range = ranges.get(0); ByteSourceRange range = ranges.get(0);
assertEquals(10, range.getStart().getOffset()); assertEquals(10, range.getStart().getOffset());
assertEquals(15, range.getEnd().getOffset()); assertEquals(15, range.getEnd().getOffset());
@ -857,7 +855,7 @@ public class MemBlockDBTest extends AbstractGenericTest {
assertEquals(0, range.getOffset()); assertEquals(0, range.getOffset());
} }
@Test @Test
public void testAddressSourceInfoForFileBytesBlock() throws Exception { public void testAddressSourceInfoForFileBytesBlock() throws Exception {
FileBytes fileBytes = createFileBytes(); FileBytes fileBytes = createFileBytes();
mem.createInitializedBlock("block", addr(100), fileBytes, 10, 50, false); mem.createInitializedBlock("block", addr(100), fileBytes, 10, 50, false);
@ -919,12 +917,13 @@ public class MemBlockDBTest extends AbstractGenericTest {
false); false);
} }
private FileBytes createFileBytes() throws IOException { private FileBytes createFileBytes() throws Exception {
byte[] bytes = new byte[256]; byte[] bytes = new byte[256];
for (int i = 0; i < 256; i++) { for (int i = 0; i < 256; i++) {
bytes[i] = (byte) i; bytes[i] = (byte) i;
} }
FileBytes fileBytes = mem.createFileBytes("test", 0, 100, new ByteArrayInputStream(bytes)); FileBytes fileBytes =
mem.createFileBytes("test", 0, 100, new ByteArrayInputStream(bytes), TaskMonitor.DUMMY);
return fileBytes; return fileBytes;
} }