GP-2687 - Filter modules used by using the classpath

This commit is contained in:
dragonmacher 2022-10-19 13:38:58 -04:00
parent 843aee1df1
commit 98927d7e4e
13 changed files with 288 additions and 321 deletions

View file

@ -36,7 +36,6 @@ import ghidra.framework.plugintool.PluginTool;
import ghidra.framework.plugintool.util.PluginException;
import ghidra.program.model.listing.Program;
import ghidra.util.TaskUtilities;
import ghidra.util.exception.AssertException;
import junit.framework.AssertionFailedError;
import utility.application.ApplicationLayout;
@ -44,22 +43,15 @@ public abstract class AbstractGhidraHeadedIntegrationTest
extends AbstractGhidraHeadlessIntegrationTest {
public AbstractGhidraHeadedIntegrationTest() {
super();
// Ensure that all headed tests use swing popups when displaying errors. Setting this
// to false would force errors to only be written to the console.
setErrorGUIEnabled(true);
}
@Override
protected ApplicationLayout createApplicationLayout() {
try {
protected ApplicationLayout createApplicationLayout() throws IOException {
return new GhidraTestApplicationLayout(new File(getTestDirectoryPath()));
}
catch (IOException e) {
throw new AssertException(e);
}
}
@Override
protected ApplicationConfiguration createApplicationConfiguration() {

View file

@ -46,7 +46,6 @@ import ghidra.program.model.symbol.Symbol;
import ghidra.program.util.*;
import ghidra.util.Msg;
import ghidra.util.classfinder.ClassSearcher;
import ghidra.util.exception.AssertException;
import ghidra.util.exception.RollbackException;
import junit.framework.AssertionFailedError;
import utility.application.ApplicationLayout;
@ -75,14 +74,9 @@ public abstract class AbstractGhidraHeadlessIntegrationTest extends AbstractDock
}
@Override
protected ApplicationLayout createApplicationLayout() {
try {
protected ApplicationLayout createApplicationLayout() throws IOException {
return new GhidraTestApplicationLayout(new File(getTestDirectoryPath()));
}
catch (IOException e) {
throw new AssertException(e);
}
}
@Override
protected ApplicationConfiguration createApplicationConfiguration() {

View file

@ -17,7 +17,6 @@ package ghidra.server.remote;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import ghidra.framework.ApplicationProperties;
@ -61,7 +60,7 @@ public class GhidraServerApplicationLayout extends ApplicationLayout {
// Modules - required to find module data files
modules = ModuleUtilities.findModules(applicationRootDirs,
ModuleUtilities.findModuleRootDirectories(applicationRootDirs, new ArrayList<>()));
ModuleUtilities.findModuleRootDirectories(applicationRootDirs));
}
}

View file

@ -16,7 +16,8 @@
package docking.framework;
import java.io.FileNotFoundException;
import java.util.*;
import java.util.Collection;
import java.util.Objects;
import generic.jar.ResourceFile;
import ghidra.framework.ApplicationProperties;
@ -24,6 +25,7 @@ import ghidra.util.SystemUtilities;
import util.CollectionUtils;
import utility.application.ApplicationLayout;
import utility.application.ApplicationUtilities;
import utility.module.ClasspathFilter;
import utility.module.ModuleUtilities;
/**
@ -62,7 +64,7 @@ public class DockingApplicationLayout extends ApplicationLayout {
* properties.
*
* @param applicationRootDirs list of application root directories which should be
* used to idenitfy modules and resources. The first entry will be treated as the
* used to identify modules and resources. The first entry will be treated as the
* installation root.
* @param applicationProperties The properties object that will be read system properties.
* @throws FileNotFoundException if there was a problem getting a user directory.
@ -81,8 +83,14 @@ public class DockingApplicationLayout extends ApplicationLayout {
// Modules
if (SystemUtilities.isInDevelopmentMode()) {
// In development mode we rely on the IDE's classpath to determine which modules to
// include, as opposed to scanning the filesystem. This prevents unrelated modules
// from being used.
modules = ModuleUtilities.findModules(applicationRootDirs,
ModuleUtilities.findModuleRootDirectories(applicationRootDirs, new ArrayList<>()));
ModuleUtilities.findModuleRootDirectories(applicationRootDirs),
new ClasspathFilter());
}
else {
modules = ModuleUtilities.findModules(applicationRootDirs, applicationRootDirs);

View file

@ -76,8 +76,6 @@ public abstract class AbstractDockingTest extends AbstractGenericTest {
new TestFailingErrorDisplayWrapper();
public AbstractDockingTest() {
super();
installNonNativeSystemClipboard();
}
@ -110,14 +108,9 @@ public abstract class AbstractDockingTest extends AbstractGenericTest {
}
@Override
protected ApplicationLayout createApplicationLayout() {
try {
protected ApplicationLayout createApplicationLayout() throws IOException {
return new GhidraTestApplicationLayout(new File(getTestDirectoryPath()));
}
catch (IOException e) {
throw new AssertException(e);
}
}
@Override
protected ApplicationConfiguration createApplicationConfiguration() {
@ -727,13 +720,9 @@ public abstract class AbstractDockingTest extends AbstractGenericTest {
}
T t = windowManager.getComponentProvider(clazz);
if (t != null) {
return t;
}
return null;
}
/**
* Searches for the first occurrence of a {@link ComponentProvider} that is an instance of
* the given <code>providerClass</code>. This method will repeat the search every
@ -2141,8 +2130,7 @@ public abstract class AbstractDockingTest extends AbstractGenericTest {
return runSwing(() -> action.isEnabledForContext(new ActionContext()));
}
public static boolean isEnabled(DockingActionIf action,
ActionContextProvider contextProvider) {
public static boolean isEnabled(DockingActionIf action, ActionContextProvider contextProvider) {
return runSwing(() -> action.isEnabledForContext(contextProvider.getActionContext(null)));
}

View file

@ -136,7 +136,14 @@ public abstract class AbstractGenericTest extends AbstractGTest {
initializeSystemProperties();
ApplicationLayout layout = test.createApplicationLayout();
ApplicationLayout layout;
try {
layout = test.createApplicationLayout();
}
catch (Exception e) {
throw new AssertException(e);
}
initializeLayout(layout);
ApplicationConfiguration configuration = test.createApplicationConfiguration();
if (initialized) {
@ -157,7 +164,7 @@ public abstract class AbstractGenericTest extends AbstractGTest {
Application.initializeApplication(layout, configuration);
}
catch (Exception e) {
throw e;
throw new AssertException(e);
}
}
@ -208,14 +215,9 @@ public abstract class AbstractGenericTest extends AbstractGTest {
initialize(this);
}
protected ApplicationLayout createApplicationLayout() {
try {
protected ApplicationLayout createApplicationLayout() throws IOException {
return new GhidraTestApplicationLayout(new File(getTestDirectoryPath()));
}
catch (IOException e) {
throw new AssertException(e);
}
}
protected ApplicationConfiguration createApplicationConfiguration() {
ApplicationConfiguration configuration = new ApplicationConfiguration();

View file

@ -52,7 +52,6 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// Application installation directory
applicationInstallationDir = findGhidraApplicationInstallationDir();
// User directories
userTempDir = ApplicationUtilities.getDefaultUserTempDir(getApplicationProperties());
userCacheDir = ApplicationUtilities.getDefaultUserCacheDir(getApplicationProperties());
@ -183,7 +182,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// These might exist if Ghidra was launched from an Eclipse project that resides
// external to the Ghidra installation.
for (String entry : System.getProperty("java.class.path", "").split(File.pathSeparator)) {
final ResourceFile classpathEntry = new ResourceFile(entry);
ResourceFile classpathEntry = new ResourceFile(entry);
// We only care about directories (skip jars)
if (!classpathEntry.isDirectory()) {
@ -193,8 +192,8 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// Skip classpath entries that live in an application root directory...we've already
// found those.
if (applicationRootDirs.stream()
.anyMatch(dir -> FileUtilities.isPathContainedWithin(
dir.getFile(false), classpathEntry.getFile(false)))) {
.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir.getFile(false),
classpathEntry.getFile(false)))) {
continue;
}

View file

@ -16,10 +16,14 @@
package ghidra;
import java.io.*;
import java.util.Collections;
import java.util.List;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
import utility.module.ClasspathFilter;
import utility.module.ModuleUtilities;
/**
* The Ghidra test application layout defines the customizable elements of the Ghidra
@ -42,7 +46,6 @@ public class GhidraTestApplicationLayout extends GhidraApplicationLayout {
*/
public GhidraTestApplicationLayout(File userSettingsDir)
throws FileNotFoundException, IOException {
super();
this.userSettingsDir = userSettingsDir;
}
@ -63,4 +66,51 @@ public class GhidraTestApplicationLayout extends GhidraApplicationLayout {
File dir = new File(getUserTempDir(), "patch");
return new ResourceFile(dir);
}
@Override
protected Map<String, GModule> findGhidraModules() throws IOException {
//
// 1) Enforces module dependencies by classpath to better control the test environment.
// 2) Add any dependent modules into the tests that are not already on the classpath. For
// example, this class adds all processor modules, as we do not use classpath
// dependencies for processor modules usage.
//
Set<String> modulePatterns = getDependentModulePatterns();
Predicate<Path> additionalPaths = path -> {
String pathString = path.toString();
return modulePatterns.stream().anyMatch(pattern -> pathString.contains(pattern));
};
Collection<ResourceFile> roots =
ModuleUtilities.findModuleRootDirectories(applicationRootDirs);
return ModuleUtilities.findModules(applicationRootDirs, roots,
new ClasspathFilter(additionalPaths));
}
/**
* Returns patterns that will be used to check against each discovered module. Matching module
* paths will be included as modules to be used during testing. By default, only modules that
* match the classpath entries are included. If your tests needs modules not referenced by the
* classpath, then you can override this method and add any module patterns needed.
*
* <p>The pattern is any desired text that will be matched against. If you wish to use path
* separators, be sure to do so in a platform-dependent manner.
*
* @return the patterns
*/
protected Set<String> getDependentModulePatterns() {
//@formatter:off
char slash = File.separatorChar;
return new HashSet<>(Set.of(
slash + "Processors" + slash,
"TestResources",
// This could easily be in a subclass, included by only those tests that need this
// entry. At the time of writing, there are 8 tests that need this module. For now,
// adding the entry here seems like the easiest thing to do.
"DemanglerGnu"
));
//@formatter:on
}
}

View file

@ -177,4 +177,5 @@ public abstract class ApplicationLayout {
public boolean inSingleJarMode() {
return false;
}
}

View file

@ -0,0 +1,73 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package utility.module;
import java.io.File;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
/**
* A predicate used to filter modules using the classpath. Only modules included in the classpath
* will pass this filter. Any modules not on the classpath may be included by calling
* {@link #ClasspathFilter(Predicate)} with a predicate that allows other module paths.
*/
public class ClasspathFilter implements Predicate<GModule> {
private Predicate<Path> additionalPaths = p -> false;
private Set<Path> cpModulePaths = new HashSet<>();
/**
* Default constructor to allow only modules on the classpath.
*/
public ClasspathFilter() {
String cp = System.getProperty("java.class.path");
String[] cpPathStrings = cp.split(File.pathSeparator);
for (String cpPathString : cpPathStrings) {
Path modulePath = ModuleUtilities.getModule(cpPathString);
if (modulePath != null) {
Path normalized = modulePath.normalize().toAbsolutePath();
cpModulePaths.add(normalized);
}
}
}
/**
* Constructor that allows any module to be included whose path passed the given predicate. If
* the predicate returns false, then a given module will only be included if it is in the
* classpath.
*
* @param additionalPaths a predicate that allows additional module paths (they do not need to
* be on the system classpath)
*/
public ClasspathFilter(Predicate<Path> additionalPaths) {
this();
this.additionalPaths = additionalPaths;
}
@Override
public boolean test(GModule m) {
ResourceFile file = m.getModuleRoot();
Path path = Path.of(file.getAbsolutePath());
Path normalized = path.normalize().toAbsolutePath();
return additionalPaths.test(normalized) || cpModulePaths.contains(normalized);
}
}

View file

@ -19,6 +19,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.util.*;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
@ -100,6 +101,21 @@ public class ModuleUtilities {
return moduleRootDirs;
}
/**
* Searches the given root directories for module root directories. Adds any discovered module
* root directories to the returned collection.
*
* <p>Note: if you need to control the type of collection used to store the module roots, then
* call {@link #findModuleRootDirectories(Collection, Collection)}.
*
* @param rootDirs The directories to look for module root directories in.
* @return a new collection with any discovered modules added.
*/
public static Collection<ResourceFile> findModuleRootDirectories(
Collection<ResourceFile> rootDirs) {
return findModuleRootDirectories(rootDirs, new ArrayList<>());
}
/**
* Searches the given root directories for module root directories. Adds any discovered module
* root directories to the given collection.
@ -146,10 +162,31 @@ public class ModuleUtilities {
public static Map<String, GModule> findModules(Collection<ResourceFile> appRootDirs,
Collection<ResourceFile> moduleRootDirs) {
Predicate<GModule> allModules = module -> true;
return findModules(appRootDirs, moduleRootDirs, allModules);
}
/**
* Searches for modules in a given collection of module root directories.
*
* @param appRootDirs The collection of application root directories associated with the the given
* list of module root directories.
* @param moduleRootDirs A collection of module root directories to search for modules in.
* @param moduleFilter a predicate used to filter modules; a given module will not be included
* when the predicate returns false.
* @return The discovered modules as a map (mapping module name to module for convenience).
*/
public static Map<String, GModule> findModules(Collection<ResourceFile> appRootDirs,
Collection<ResourceFile> moduleRootDirs, Predicate<GModule> moduleFilter) {
Map<String, GModule> map = new TreeMap<>();
for (ResourceFile moduleRoot : moduleRootDirs) {
GModule gModule = new GModule(appRootDirs, moduleRoot);
if (!moduleFilter.test(gModule)) {
continue;
}
if (map.put(moduleRoot.getName(), gModule) != null) {
StringBuilder collided = new StringBuilder();
for (ResourceFile collideRoot : moduleRootDirs) {
@ -197,8 +234,8 @@ public class ModuleUtilities {
String[] binaryPathTokens = BINARY_PATH.split(":");
List<ResourceFile> binDirectories = new ArrayList<>();
for (GModule module : modules.values()) {
Arrays.stream(binaryPathTokens).forEach(
token -> module.collectExistingModuleDirs(binDirectories, token));
Arrays.stream(binaryPathTokens)
.forEach(token -> module.collectExistingModuleDirs(binDirectories, token));
}
return binDirectories;
}

View file

@ -60,6 +60,7 @@ dependencies {
testImplementation project(path: ':Base', configuration: 'integrationTestArtifacts')
testImplementation project(path: ':FunctionGraph', configuration: 'testArtifacts')
testImplementation project(path: ':PDB', configuration: 'testArtifacts')
testImplementation project(path: ':GnuDemangler', configuration: 'testArtifacts')
}
// For Java 9, we must explicitly export references to the internal classes we are using.

View file

@ -1,177 +0,0 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ghidra.pcodeCPort.slgh_compile;
import static org.junit.Assert.assertEquals;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.Before;
import org.junit.Test;
import ghidra.app.plugin.languages.sleigh.ConstructorEntryVisitor;
import ghidra.app.plugin.languages.sleigh.SleighLanguages;
import ghidra.app.plugin.processors.sleigh.*;
import ghidra.app.plugin.processors.sleigh.pattern.DisjointPattern;
import ghidra.app.plugin.processors.sleigh.symbol.SubtableSymbol;
import ghidra.program.model.lang.LanguageID;
import ghidra.test.AbstractGhidraHeadlessIntegrationTest;
public class WithBlockTest extends AbstractGhidraHeadlessIntegrationTest {
protected static boolean setupDone;
protected static LanguageID langID;
protected static SleighLanguageProvider provider;
protected static SleighLanguage lang;
@Before
public void setUp() throws Exception {
if (!setupDone) {
langID = new LanguageID("TestWith:BE:32:default");
provider = new SleighLanguageProvider();
lang = (SleighLanguage) provider.getLanguage(langID);
setupDone = true;
}
}
protected String opnd(int n) {
return "\n" + (char) ('A' + n);
}
protected Pair<DisjointPattern, Constructor> findConstructor(String table, String firstPrint) {
AtomicReference<DisjointPattern> fpat = new AtomicReference<>(null);
AtomicReference<Constructor> fcon = new AtomicReference<>(null);
SleighLanguages.traverseConstructors(lang, new ConstructorEntryVisitor() {
@Override
public int visit(SubtableSymbol subtable, DisjointPattern pattern, Constructor cons) {
if (table.equals(subtable.getName()) &&
firstPrint.equals(cons.getPrintPieces().get(0))) {
if (null != fpat.get()) {
throw new AssertionError("Multiple constructors found. " +
"Write the test slaspec such that no two constructors in the same " +
"table share the same first printpiece.");
}
fpat.set(pattern);
fcon.set(cons);
}
return CONTINUE;
}
});
if (null == fpat.get()) {
throw new AssertionError(
"No such constructor found: " + table + ":" + firstPrint + "...");
}
return new ImmutablePair<>(fpat.get(), fcon.get());
}
@Test
public void testNOP1_NoWith() {
Pair<DisjointPattern, Constructor> NOP1 = findConstructor("instruction", "NOP1");
assertEquals("ins:SS:XF:XX:XX:XX", NOP1.getLeft().toString());
assertEquals(0, NOP1.getRight().getContextChanges().size());
}
@Test
public void testRel8addr_WithTabRel8_WithChgPhase3() {
Pair<DisjointPattern, Constructor> Rel8addr = findConstructor("Rel8", opnd(1));
assertEquals("always", Rel8addr.getLeft().toString());
assertEquals(1, Rel8addr.getRight().getContextChanges().size());
assertEquals("ctx&F0:00:00:00 := 0x3( << 28)",
Rel8addr.getRight().getContextChanges().get(0).toString());
}
@Test
public void testOP1r0_WithTabOp1() {
Pair<DisjointPattern, Constructor> OP1r0 = findConstructor("OP1", "r0");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:X0:XX:XX:XX)", OP1r0.getLeft().toString());
assertEquals(1, OP1r0.getRight().getContextChanges().size());
assertEquals("ctx&F0:00:00:00 := 0x2( << 28)",
OP1r0.getRight().getContextChanges().get(0).toString());
}
@Test
public void testOP1r1_WithPatPhase1() {
Pair<DisjointPattern, Constructor> OP1r1 = findConstructor("OP1", "r1");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:X1:XX:XX:XX)", OP1r1.getLeft().toString());
assertEquals(1, OP1r1.getRight().getContextChanges().size());
assertEquals("ctx&F0:00:00:00 := 0x2( << 28)",
OP1r1.getRight().getContextChanges().get(0).toString());
}
@Test
public void testOP1r2_WithChgPhase2() {
Pair<DisjointPattern, Constructor> OP1r2 = findConstructor("OP1", "r2");
assertEquals("ins:X2:XX:XX:XX", OP1r2.getLeft().toString());
assertEquals(2, OP1r2.getRight().getContextChanges().size());
assertEquals("ctx&F0:00:00:00 := 0x2( << 28)",
OP1r2.getRight().getContextChanges().get(0).toString());
assertEquals("ctx&F0:00:00:00 := 0x1( << 28)",
OP1r2.getRight().getContextChanges().get(1).toString());
}
@Test
public void testDSTr0_WithPatPhase1_WithTabDST() {
Pair<DisjointPattern, Constructor> DSTr0 = findConstructor("DST", "r0");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:SS:0X:XX:XX:XX)", DSTr0.getLeft().toString());
assertEquals(0, DSTr0.getRight().getContextChanges().size());
}
@Test
public void testOP1r3_WithPatPhase1_WithTabOP1() {
Pair<DisjointPattern, Constructor> OP1r3 = findConstructor("OP1", "r3");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:X3:XX:XX:XX)", OP1r3.getLeft().toString());
assertEquals(0, OP1r3.getRight().getContextChanges().size());
}
@Test
public void testOP2r1_WithPatPhase1_WithTabOP1() {
Pair<DisjointPattern, Constructor> OP2r1 = findConstructor("OP2", "r1");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:1X:XX:XX:XX)", OP2r1.getLeft().toString());
assertEquals(0, OP2r1.getRight().getContextChanges().size());
}
@Test
public void testOP2r0_WithPatPhase1_WithTabOP2() {
Pair<DisjointPattern, Constructor> OP2r0 = findConstructor("OP2", "r0");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:0X:XX:XX:XX)", OP2r0.getLeft().toString());
assertEquals(0, OP2r0.getRight().getContextChanges().size());
}
@Test
public void testNOP2_NoWith() {
Pair<DisjointPattern, Constructor> NOP2 = findConstructor("instruction", "NOP2");
assertEquals("ins:SS:XE:XX:XX:XX", NOP2.getLeft().toString());
assertEquals(0, NOP2.getRight().getContextChanges().size());
}
@Test
public void testADD_NoWith() {
Pair<DisjointPattern, Constructor> ADD = findConstructor("instruction", "ADD");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:SS:X0:XX:XX:XX)", ADD.getLeft().toString());
assertEquals(0, ADD.getRight().getContextChanges().size());
}
@Test
public void testLD_NoWith() {
Pair<DisjointPattern, Constructor> LD = findConstructor("instruction", "LD");
assertEquals("cmb:(ctx:1X:XX:XX:XX,ins:SS:X8:XX:XX:XX)", LD.getLeft().toString());
assertEquals(0, LD.getRight().getContextChanges().size());
}
// TODO: Explicit override of subtable in with
}