GP-2687 - Filter modules used by using the classpath

This commit is contained in:
dragonmacher 2022-10-19 13:38:58 -04:00
parent 843aee1df1
commit 98927d7e4e
13 changed files with 288 additions and 321 deletions

View file

@ -35,7 +35,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
/**
* Constructs a new Ghidra application layout object.
*
*
* @throws FileNotFoundException if there was a problem getting a user
* directory.
* @throws IOException if there was a problem getting the application
@ -52,7 +52,6 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// Application installation directory
applicationInstallationDir = findGhidraApplicationInstallationDir();
// User directories
userTempDir = ApplicationUtilities.getDefaultUserTempDir(getApplicationProperties());
userCacheDir = ApplicationUtilities.getDefaultUserCacheDir(getApplicationProperties());
@ -76,7 +75,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
* <p>
* This is used when something external to Ghidra needs Ghidra's layout
* (like the Eclipse GhidraDevPlugin).
*
*
* @param applicationInstallationDir The application installation directory.
* @throws FileNotFoundException if there was a problem getting a user
* directory.
@ -101,21 +100,21 @@ public class GhidraApplicationLayout extends ApplicationLayout {
userCacheDir = ApplicationUtilities.getDefaultUserCacheDir(getApplicationProperties());
userSettingsDir = ApplicationUtilities.getDefaultUserSettingsDir(getApplicationProperties(),
getApplicationInstallationDir());
// Extensions
extensionInstallationDirs = findExtensionInstallationDirectories();
extensionArchiveDir = findExtensionArchiveDirectory();
// Patch directory
patchDir = findPatchDirectory();
// Modules
modules = findGhidraModules();
}
/**
* Finds the application root directories for this application layout.
*
*
* @return A collection of the application root directories for this layout.
*/
protected Collection<ResourceFile> findGhidraApplicationRootDirs() {
@ -125,7 +124,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
/**
* Finds the application installation directory for this Ghidra application
* layout.
*
*
* @return The application installation directory for this Ghidra
* application layout. Could be null if there is no application
* installation directory.
@ -144,7 +143,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
/**
* Finds the modules for this Ghidra application layout.
*
*
* @return The modules for this Ghidra application layout.
* @throws IOException if there was a problem finding the modules on disk.
*/
@ -161,7 +160,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
if (extensionModuleDirs != null) {
for (File extensionModuleDir : extensionModuleDirs) {
// Skip extensions that live in an application root directory...we've already
// Skip extensions that live in an application root directory...we've already
// found those.
if (applicationRootDirs.stream()
.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir.getFile(false),
@ -183,7 +182,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// These might exist if Ghidra was launched from an Eclipse project that resides
// external to the Ghidra installation.
for (String entry : System.getProperty("java.class.path", "").split(File.pathSeparator)) {
final ResourceFile classpathEntry = new ResourceFile(entry);
ResourceFile classpathEntry = new ResourceFile(entry);
// We only care about directories (skip jars)
if (!classpathEntry.isDirectory()) {
@ -193,8 +192,8 @@ public class GhidraApplicationLayout extends ApplicationLayout {
// Skip classpath entries that live in an application root directory...we've already
// found those.
if (applicationRootDirs.stream()
.anyMatch(dir -> FileUtilities.isPathContainedWithin(
dir.getFile(false), classpathEntry.getFile(false)))) {
.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir.getFile(false),
classpathEntry.getFile(false)))) {
continue;
}
@ -234,7 +233,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
* <ul>
* <li><code>[application root]/Extensions/Ghidra</code></li>
* </ul>
*
*
* @return the archive folder, or null if can't be determined
*/
protected ResourceFile findExtensionArchiveDirectory() {
@ -250,23 +249,23 @@ public class GhidraApplicationLayout extends ApplicationLayout {
}
/**
* Returns a prioritized list of directories where Ghidra extensions are installed. These
* Returns a prioritized list of directories where Ghidra extensions are installed. These
* should be at the following locations:<br>
* <ul>
* <li><code>[user settings dir]/Extensions</code></li>
* <li><code>[application install dir]/Ghidra/Extensions</code></li>
* <li><code>ghidra/Ghidra/Extensions</code> (development mode)</li>
* </ul>
*
*
* @return the install folder, or null if can't be determined
*/
protected List<ResourceFile> findExtensionInstallationDirectories() {
List<ResourceFile> dirs = new ArrayList<>();
// Would like to find a better way to do this, but for the moment this seems the
// only solution. We want to get the 'Extensions' directory in ghidra, but there's
// no way to retrieve that directory directly. We can only get the full set of
// only solution. We want to get the 'Extensions' directory in ghidra, but there's
// no way to retrieve that directory directly. We can only get the full set of
// application root dirs and search for it, hoping we don't encounter one with the
// name 'Extensions' in one of the other root dirs.
if (SystemUtilities.isInDevelopmentMode()) {

View file

@ -16,13 +16,17 @@
package ghidra;
import java.io.*;
import java.util.Collections;
import java.util.List;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
import utility.module.ClasspathFilter;
import utility.module.ModuleUtilities;
/**
* The Ghidra test application layout defines the customizable elements of the Ghidra
* The Ghidra test application layout defines the customizable elements of the Ghidra
* application's directory structure when running a test.
* <p>
* This layout exists because tests often need to provide their own user settings
@ -35,14 +39,13 @@ public class GhidraTestApplicationLayout extends GhidraApplicationLayout {
* directory.
* <p>
* This layout is useful when running Ghidra tests.
*
*
* @param userSettingsDir The custom user settings directory to use.
* @throws FileNotFoundException if there was a problem getting a user directory.
* @throws IOException if there was a problem getting the application properties.
*/
public GhidraTestApplicationLayout(File userSettingsDir)
throws FileNotFoundException, IOException {
super();
this.userSettingsDir = userSettingsDir;
}
@ -63,4 +66,51 @@ public class GhidraTestApplicationLayout extends GhidraApplicationLayout {
File dir = new File(getUserTempDir(), "patch");
return new ResourceFile(dir);
}
@Override
protected Map<String, GModule> findGhidraModules() throws IOException {
//
// 1) Enforces module dependencies by classpath to better control the test environment.
// 2) Add any dependent modules into the tests that are not already on the classpath. For
// example, this class adds all processor modules, as we do not use classpath
// dependencies for processor modules usage.
//
Set<String> modulePatterns = getDependentModulePatterns();
Predicate<Path> additionalPaths = path -> {
String pathString = path.toString();
return modulePatterns.stream().anyMatch(pattern -> pathString.contains(pattern));
};
Collection<ResourceFile> roots =
ModuleUtilities.findModuleRootDirectories(applicationRootDirs);
return ModuleUtilities.findModules(applicationRootDirs, roots,
new ClasspathFilter(additionalPaths));
}
/**
* Returns patterns that will be used to check against each discovered module. Matching module
* paths will be included as modules to be used during testing. By default, only modules that
* match the classpath entries are included. If your tests needs modules not referenced by the
* classpath, then you can override this method and add any module patterns needed.
*
* <p>The pattern is any desired text that will be matched against. If you wish to use path
* separators, be sure to do so in a platform-dependent manner.
*
* @return the patterns
*/
protected Set<String> getDependentModulePatterns() {
//@formatter:off
char slash = File.separatorChar;
return new HashSet<>(Set.of(
slash + "Processors" + slash,
"TestResources",
// This could easily be in a subclass, included by only those tests that need this
// entry. At the time of writing, there are 8 tests that need this module. For now,
// adding the entry here seems like the easiest thing to do.
"DemanglerGnu"
));
//@formatter:on
}
}

View file

@ -25,11 +25,11 @@ import ghidra.framework.GModule;
import utilities.util.FileUtilities;
/**
* The Application Layout base class defines the customizable elements of the application's
* The Application Layout base class defines the customizable elements of the application's
* directory structure. Create a subclass to define a custom layout.
* <p>
* If a layout changes in a significant way, the
* {@link ApplicationProperties#APPLICATION_LAYOUT_VERSION_PROPERTY} should be incremented so
* If a layout changes in a significant way, the
* {@link ApplicationProperties#APPLICATION_LAYOUT_VERSION_PROPERTY} should be incremented so
* external things like Eclipse GhidraDev know to look in different places for things.
*/
public abstract class ApplicationLayout {
@ -47,7 +47,7 @@ public abstract class ApplicationLayout {
/**
* Gets the application properties from the application layout
*
*
* @return The application properties. Should never be null.
*/
public final ApplicationProperties getApplicationProperties() {
@ -56,7 +56,7 @@ public abstract class ApplicationLayout {
/**
* Gets the application root directories from the application layout.
*
*
* @return A collection of application root directories (or null if not set).
*/
public final Collection<ResourceFile> getApplicationRootDirs() {
@ -65,7 +65,7 @@ public abstract class ApplicationLayout {
/**
* Gets the application installation directory from the application layout.
*
*
* @return The application installation directory (or null if not set).
*/
public final ResourceFile getApplicationInstallationDir() {
@ -74,7 +74,7 @@ public abstract class ApplicationLayout {
/**
* Gets the application's modules from the application layout.
*
*
* @return The application's modules as a map (mapping module name to module for convenience).
*/
public final Map<String, GModule> getModules() {
@ -83,7 +83,7 @@ public abstract class ApplicationLayout {
/**
* Gets the user temp directory from the application layout.
*
*
* @return The user temp directory (or null if not set).
*/
public final File getUserTempDir() {
@ -92,7 +92,7 @@ public abstract class ApplicationLayout {
/**
* Gets the user cache directory from the application layout.
*
*
* @return The user cache directory (or null if not set).
*/
public final File getUserCacheDir() {
@ -101,7 +101,7 @@ public abstract class ApplicationLayout {
/**
* Gets the user settings directory from the application layout.
*
*
* @return The user settings directory (or null if not set).
*/
public final File getUserSettingsDir() {
@ -110,10 +110,10 @@ public abstract class ApplicationLayout {
/**
* Returns the directory where archived application Extensions are stored.
*
* @return the application Extensions archive directory. Could be null if the
*
* @return the application Extensions archive directory. Could be null if the
* {@link ApplicationLayout} does not support application Extensions.
*
*
*/
public final ResourceFile getExtensionArchiveDir() {
return extensionArchiveDir;
@ -121,7 +121,7 @@ public abstract class ApplicationLayout {
/**
* Returns an {@link List ordered list} of the application Extensions installation directories.
*
*
* @return an {@link List ordered list} of the application Extensions installation directories.
* Could be empty if the {@link ApplicationLayout} does not support application Extensions.
*/
@ -140,7 +140,7 @@ public abstract class ApplicationLayout {
/**
* Creates the application's user directories (or ensures they already exist).
*
*
* @throws IOException if there was a problem creating the application's user directories.
*/
public final void createUserDirs() throws IOException {
@ -168,13 +168,14 @@ public abstract class ApplicationLayout {
}
/**
* Checks whether or not the application is using a "single jar" layout. Custom application
* layouts that extend this class can override this method once they determine they are in
* Checks whether or not the application is using a "single jar" layout. Custom application
* layouts that extend this class can override this method once they determine they are in
* single jar mode.
*
*
* @return true if the application is using a "single jar" layout; otherwise, false.
*/
public boolean inSingleJarMode() {
return false;
}
}

View file

@ -0,0 +1,73 @@
/* ###
* IP: GHIDRA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package utility.module;
import java.io.File;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
/**
* A predicate used to filter modules using the classpath. Only modules included in the classpath
* will pass this filter. Any modules not on the classpath may be included by calling
* {@link #ClasspathFilter(Predicate)} with a predicate that allows other module paths.
*/
public class ClasspathFilter implements Predicate<GModule> {
private Predicate<Path> additionalPaths = p -> false;
private Set<Path> cpModulePaths = new HashSet<>();
/**
* Default constructor to allow only modules on the classpath.
*/
public ClasspathFilter() {
String cp = System.getProperty("java.class.path");
String[] cpPathStrings = cp.split(File.pathSeparator);
for (String cpPathString : cpPathStrings) {
Path modulePath = ModuleUtilities.getModule(cpPathString);
if (modulePath != null) {
Path normalized = modulePath.normalize().toAbsolutePath();
cpModulePaths.add(normalized);
}
}
}
/**
* Constructor that allows any module to be included whose path passed the given predicate. If
* the predicate returns false, then a given module will only be included if it is in the
* classpath.
*
* @param additionalPaths a predicate that allows additional module paths (they do not need to
* be on the system classpath)
*/
public ClasspathFilter(Predicate<Path> additionalPaths) {
this();
this.additionalPaths = additionalPaths;
}
@Override
public boolean test(GModule m) {
ResourceFile file = m.getModuleRoot();
Path path = Path.of(file.getAbsolutePath());
Path normalized = path.normalize().toAbsolutePath();
return additionalPaths.test(normalized) || cpModulePaths.contains(normalized);
}
}

View file

@ -19,6 +19,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.util.*;
import java.util.function.Predicate;
import generic.jar.ResourceFile;
import ghidra.framework.GModule;
@ -39,14 +40,14 @@ public class ModuleUtilities {
public static final String MODULE_LIST = "MODULE_LIST";
/**
* How many directories deep to look for module directories, starting from an application root
* How many directories deep to look for module directories, starting from an application root
* directory. For example, 3 would pick up modules as deep as: root/category/category/module
*/
private static final int MAX_MODULE_DEPTH = 3;
/**
* Checks if the given directory is a module.
*
*
* @param dir the directory to check.
* @return true if the given directory is a module
*/
@ -56,7 +57,7 @@ public class ModuleUtilities {
/**
* Returns true if the given path is a module root directory.
*
*
* @param path the path to check
* @return true if the given path is a module root directory.
*/
@ -68,14 +69,14 @@ public class ModuleUtilities {
/**
* Searches the given root directory for module root directories. Adds any discovered module
* root directories to the given collection.
*
*
* @param rootDir The directory to start looking for module root directories in.
* @param moduleRootDirs A collection to add discovered module root directories to.
* @return The given collection with any discovered modules added.
*/
public static Collection<ResourceFile> findModuleRootDirectories(ResourceFile rootDir,
Collection<ResourceFile> moduleRootDirs) {
// look for any external GPL modules
// look for any external GPL modules
findModuleRootDirectoriesHelper(new ResourceFile(rootDir, "../GPL"), moduleRootDirs,
MAX_MODULE_DEPTH);
return findModuleRootDirectoriesHelper(rootDir, moduleRootDirs, MAX_MODULE_DEPTH);
@ -100,10 +101,25 @@ public class ModuleUtilities {
return moduleRootDirs;
}
/**
* Searches the given root directories for module root directories. Adds any discovered module
* root directories to the returned collection.
*
* <p>Note: if you need to control the type of collection used to store the module roots, then
* call {@link #findModuleRootDirectories(Collection, Collection)}.
*
* @param rootDirs The directories to look for module root directories in.
* @return a new collection with any discovered modules added.
*/
public static Collection<ResourceFile> findModuleRootDirectories(
Collection<ResourceFile> rootDirs) {
return findModuleRootDirectories(rootDirs, new ArrayList<>());
}
/**
* Searches the given root directories for module root directories. Adds any discovered module
* root directories to the given collection.
*
*
* @param rootDirs The directories to look for module root directories in.
* @param moduleRootDirs A collection to add discovered module root directories to.
* @return The given collection with any discovered modules added.
@ -120,7 +136,7 @@ public class ModuleUtilities {
* Searches the given jar root directory for module root directories. Uses a "module list"
* file to locate the module root directories. Adds any discovered module root directories
* to the given collection.
*
*
* @param rootDir The jar directory to start looking for module root directories in.
* @param moduleRootDirs A collection to add discovered module root directories to.
* @return The given collection with any discovered modules added.
@ -137,7 +153,7 @@ public class ModuleUtilities {
/**
* Searches for modules in a given collection of module root directories.
*
*
* @param appRootDirs The collection of application root directories associated with the the given
* list of module root directories.
* @param moduleRootDirs A collection of module root directories to search for modules in.
@ -146,10 +162,31 @@ public class ModuleUtilities {
public static Map<String, GModule> findModules(Collection<ResourceFile> appRootDirs,
Collection<ResourceFile> moduleRootDirs) {
Predicate<GModule> allModules = module -> true;
return findModules(appRootDirs, moduleRootDirs, allModules);
}
/**
* Searches for modules in a given collection of module root directories.
*
* @param appRootDirs The collection of application root directories associated with the the given
* list of module root directories.
* @param moduleRootDirs A collection of module root directories to search for modules in.
* @param moduleFilter a predicate used to filter modules; a given module will not be included
* when the predicate returns false.
* @return The discovered modules as a map (mapping module name to module for convenience).
*/
public static Map<String, GModule> findModules(Collection<ResourceFile> appRootDirs,
Collection<ResourceFile> moduleRootDirs, Predicate<GModule> moduleFilter) {
Map<String, GModule> map = new TreeMap<>();
for (ResourceFile moduleRoot : moduleRootDirs) {
GModule gModule = new GModule(appRootDirs, moduleRoot);
if (!moduleFilter.test(gModule)) {
continue;
}
if (map.put(moduleRoot.getName(), gModule) != null) {
StringBuilder collided = new StringBuilder();
for (ResourceFile collideRoot : moduleRootDirs) {
@ -167,7 +204,7 @@ public class ModuleUtilities {
/**
* Gets the "lib" directories from the given modules.
*
*
* @param modules The modules to get the lib directories of.
* @return A collection of lib directories from the given modules.
*/
@ -189,7 +226,7 @@ public class ModuleUtilities {
/**
* Gets the directory locations of the .class files and resources from the given modules.
*
*
* @param modules The modules to get the compiled .class and resources directories of.
* @return A collection of directories containing classes and resources from the given modules.
*/
@ -197,8 +234,8 @@ public class ModuleUtilities {
String[] binaryPathTokens = BINARY_PATH.split(":");
List<ResourceFile> binDirectories = new ArrayList<>();
for (GModule module : modules.values()) {
Arrays.stream(binaryPathTokens).forEach(
token -> module.collectExistingModuleDirs(binDirectories, token));
Arrays.stream(binaryPathTokens)
.forEach(token -> module.collectExistingModuleDirs(binDirectories, token));
}
return binDirectories;
}
@ -215,10 +252,10 @@ public class ModuleUtilities {
* <br>
* <br>and false for these paths:
* <br>
* <br>
* <br>
* <code>/some/random/path</code><br>
* <code>/some/dir/features/</code>
*
*
* @param pathName the path name to check
* @return true if the given path is parented by a module root directory.
* @see #isModuleDirectory(Path)
@ -228,7 +265,7 @@ public class ModuleUtilities {
}
/**
* Returns the path of the module containing the given path string, if it is parented by a
* Returns the path of the module containing the given path string, if it is parented by a
* module root directory.
* <p>
* For example, given a module path of <code>/some/dir/features/cool_module/</code>, then this
@ -240,10 +277,10 @@ public class ModuleUtilities {
* <br>
* <br>and null for these paths:
* <br>
* <br>
* <br>
* <code>/some/random/path</code><br>
* <code>/some/dir/features/</code>
*
*
* @param pathName the path name to check
* @return the module root directory; null if the path is not in a module
* @see #isModuleDirectory(Path)
@ -271,18 +308,18 @@ public class ModuleUtilities {
/**
* Returns a file that is the root folder of the repository containing the given file. 'Root'
* here means a folder that contains a repository folder. As an example, given a repo
* here means a folder that contains a repository folder. As an example, given a repo
* structure of:
*
*
* <p><code>/userdir/repoRoot/repoDir/.git</code><br>
*
*
* <p>then this method, given will produce the following results (input -&gt; output):<br>
*
*
* <p><code>/userdir/repoRoot/repoDir/.git -&gt; /userdir/repoRoot</code>
* <br><code>/userdir/repoRoot/repoDir -&gt; /userdir/repoRoot</code>
* <br><code>/userdir/repoRoot -&gt; /userdir/repoRoot</code>
*
*
*
*
* @param f the child file of the desired repo
* @return a file that is the root folder of the repository containing the given file; null
* if the given file is not under a repo directory or itself a repo root
@ -311,16 +348,16 @@ public class ModuleUtilities {
}
/**
* Returns a file that is the repository folder containing the given file. As an example,
* Returns a file that is the repository folder containing the given file. As an example,
* given a repo structure of:
*
*
* <p><code>/userdir/repoRoot/repoDir/.git</code><br>
*
*
* <p>then this method, given will produce the following results (input -&gt; output):<br>
*
*
* <p><code>/userdir/repoRoot/repoDir/.git -&gt; /userdir/repoRoot/repoDir</code>
* <br><code>/userdir/repoRoot/repoDir -&gt; /userdir/repoRoot/repoDir</code>
*
*
* @param f the child file of the desired repo
* @return a file that is the repo folder of the repository containing the given file; null
* if the given file is not under a repo directory
@ -339,7 +376,7 @@ public class ModuleUtilities {
/**
* Checks to see if the given {@link GModule module} is external to the Ghidra installation
* directory
*
*
* @param module the module to check
* @param layout Ghidra's layout
* @return true if the given {@link GModule module} is external to the Ghidra installation