GP-3569 - Cleanup of Extension management

Ghidra/Framework/Utility/src/main/java/generic/jar/ResourceFile.java

@@ -20,6 +20,8 @@ import java.net.*;
 import java.util.HashMap;
 import java.util.Map;
 
+import utilities.util.FileUtilities;
+
 /**
  * Class for representing file object regardless of whether they are actual files in the file system or
  * or files stored inside of a jar file.  This class provides most all the same capabilities as the
@@ -30,7 +32,7 @@ import java.util.Map;
 public class ResourceFile implements Comparable<ResourceFile> {
 	private static final String JAR_FILE_PREFIX = "jar:file:";
 	private Resource resource;
-	private static Map<String, JarResource> jarRootsMap = new HashMap<String, JarResource>();
+	private static Map<String, JarResource> jarRootsMap = new HashMap<>();
 
 	/**
 	 * Construct a ResourceFile that represents a normal file in the file system.
@@ -121,6 +123,7 @@ public class ResourceFile implements Comparable<ResourceFile> {
 	/**
 	 * Returns the canonical file path for this file.
 	 * @return the absolute file path for this file.
+	 * @throws IOException if an exception is thrown getting the canonical path
 	 */
 	public String getCanonicalPath() throws IOException {
 		return resource.getCanonicalPath();
@@ -128,7 +131,7 @@ public class ResourceFile implements Comparable<ResourceFile> {
 
 	/**
 	 * Returns a array of ResourceFiles if this ResourceFile is a directory. Otherwise return null.
-	 * @return  the child ResourceFiles if this is a directory, null otherwise.
+	 * @return the child ResourceFiles if this is a directory, null otherwise.
 	 */
 	public ResourceFile[] listFiles() {
 		return resource.listFiles();
@@ -190,7 +193,7 @@ public class ResourceFile implements Comparable<ResourceFile> {
 	 * contents. 
 	 * @return an InputStream for the file's contents.
 	 * @throws FileNotFoundException if the file does not exist.
-	 * @throws IOException 
+	 * @throws IOException if an exception occurs creating the input stream
 	 */
 	public InputStream getInputStream() throws FileNotFoundException, IOException {
 		return resource.getInputStream();
@@ -329,4 +332,13 @@ public class ResourceFile implements Comparable<ResourceFile> {
 	public URI toURI() {
 		return resource.toURI();
 	}
+
+	/**
+	 * Returns true if this file's path contains the entire path of the given file.
+	 * @param otherFile the other file to check
+	 * @return true if this file's path contains the entire path of the given file.
+	 */
+	public boolean containsPath(ResourceFile otherFile) {
+		return FileUtilities.isPathContainedWithin(getFile(false), otherFile.getFile(false));
+	}
 }

Ghidra/Framework/Utility/src/main/java/ghidra/GhidraApplicationLayout.java

@@ -155,27 +155,20 @@ public class GhidraApplicationLayout extends ApplicationLayout {
 
 		// Find installed extension modules
 		for (ResourceFile extensionInstallDir : extensionInstallationDirs) {
-			File[] extensionModuleDirs =
-				extensionInstallDir.getFile(false).listFiles(d -> d.isDirectory());
-			if (extensionModuleDirs != null) {
-				for (File extensionModuleDir : extensionModuleDirs) {
 
-					// Skip extensions that live in an application root directory...we've already
-					// found those.
-					if (applicationRootDirs.stream()
-							.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir.getFile(false),
-								extensionModuleDir))) {
-						continue;
-					}
-					// Skip extensions slated for cleanup
-					if (new File(extensionModuleDir, ModuleUtilities.MANIFEST_FILE_NAME_UNINSTALLED)
-							.exists()) {
-						continue;
-					}
-
-					moduleRootDirectories.add(new ResourceFile(extensionModuleDir));
+			FileUtilities.forEachFile(extensionInstallDir, extensionDir -> {
+				// Skip extensions in an application root directory... already found those.
+				if (FileUtilities.isPathContainedWithin(applicationRootDirs, extensionDir)) {
+					return;
 				}
-			}
+
+				// Skip extensions slated for cleanup
+				if (ModuleUtilities.isUninstalled(extensionDir)) {
+					return;
+				}
+
+				moduleRootDirectories.add(extensionDir);
+			});
 		}
 
 		// Examine the classpath to look for modules outside of the application root directories.
@@ -189,11 +182,8 @@ public class GhidraApplicationLayout extends ApplicationLayout {
 				continue;
 			}
 
-			// Skip classpath entries that live in an application root directory...we've already
-			// found those.
-			if (applicationRootDirs.stream()
-					.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir.getFile(false),
-						classpathEntry.getFile(false)))) {
+			// Skip extensions in an application root directory... already found those.
+			if (FileUtilities.isPathContainedWithin(applicationRootDirs, classpathEntry)) {
 				continue;
 			}
 
@@ -231,7 +221,7 @@ public class GhidraApplicationLayout extends ApplicationLayout {
 	 * Returns the directory where all Ghidra extension archives are stored.
 	 * This should be at the following location:<br>
 	 * <ul>
-	 * <li><code>[application root]/Extensions/Ghidra</code></li>
+	 * <li><code>{install dir}/Extensions/Ghidra</code></li>
 	 * </ul>
 	 *
 	 * @return the archive folder, or null if can't be determined
@@ -253,8 +243,8 @@ public class GhidraApplicationLayout extends ApplicationLayout {
 	 * should be at the following locations:<br>
 	 * <ul>
 	 * <li><code>[user settings dir]/Extensions</code></li>
-	 * <li><code>[application install dir]/Ghidra/Extensions</code></li>
-	 * <li><code>ghidra/Ghidra/Extensions</code> (development mode)</li>
+	 * <li><code>[application install dir]/Ghidra/Extensions</code> (Release Mode)</li>
+	 * <li><code>ghidra/Ghidra/Extensions</code> (Development Mode)</li>
 	 * </ul>
 	 *
 	 * @return the install folder, or null if can't be determined
@@ -262,21 +252,16 @@ public class GhidraApplicationLayout extends ApplicationLayout {
 	protected List<ResourceFile> findExtensionInstallationDirectories() {
 
 		List<ResourceFile> dirs = new ArrayList<>();
+		dirs.add(new ResourceFile(new File(userSettingsDir, "Extensions")));
 
-		// Would like to find a better way to do this, but for the moment this seems the
-		// only solution. We want to get the 'Extensions' directory in ghidra, but there's
-		// no way to retrieve that directory directly. We can only get the full set of
-		// application root dirs and search for it, hoping we don't encounter one with the
-		// name 'Extensions' in one of the other root dirs.
 		if (SystemUtilities.isInDevelopmentMode()) {
 			ResourceFile rootDir = getApplicationRootDirs().iterator().next();
 			File temp = new File(rootDir.getFile(false), "Extensions");
 			if (temp.exists()) {
-				dirs.add(new ResourceFile(temp));
+				dirs.add(new ResourceFile(temp)); // ghidra/Ghidra/Extensions
 			}
 		}
 		else {
-			dirs.add(new ResourceFile(new File(userSettingsDir, "Extensions")));
 			dirs.add(new ResourceFile(applicationInstallationDir, "Ghidra/Extensions"));
 		}
 

Ghidra/Framework/Utility/src/main/java/ghidra/GhidraLauncher.java

@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
 import generic.jar.ResourceFile;
 import ghidra.framework.GModule;
 import ghidra.util.SystemUtilities;
+import utilities.util.FileUtilities;
 import utility.application.ApplicationLayout;
 import utility.module.ModuleUtilities;
 
@@ -115,7 +116,7 @@ public class GhidraLauncher {
 
 		// Get application layout
 		GhidraApplicationLayout layout = new GhidraApplicationLayout();
-		
+
 		// Get the classpath
 		List<String> classpathList = buildClasspath(layout);
 
@@ -148,6 +149,7 @@ public class GhidraLauncher {
 				addModuleJarPaths(classpathList, modules);
 			}
 
+			addExtensionJarPaths(classpathList, modules, layout);
 			addExternalJarPaths(classpathList, layout.getApplicationRootDirs());
 		}
 		else {
@@ -185,7 +187,7 @@ public class GhidraLauncher {
 	 * @param modules The modules to get the bin directories of.
 	 */
 	private static void addModuleBinPaths(List<String> pathList, Map<String, GModule> modules) {
-		Collection<ResourceFile> dirs = ModuleUtilities.getModuleBinDirectories(modules);
+		Collection<ResourceFile> dirs = ModuleUtilities.getModuleBinDirectories(modules.values());
 		dirs.forEach(d -> pathList.add(d.getAbsolutePath()));
 	}
 
@@ -196,10 +198,45 @@ public class GhidraLauncher {
 	 * @param modules The modules to get the jars of.
 	 */
 	private static void addModuleJarPaths(List<String> pathList, Map<String, GModule> modules) {
-		Collection<ResourceFile> dirs = ModuleUtilities.getModuleLibDirectories(modules);
+		Collection<ResourceFile> dirs = ModuleUtilities.getModuleLibDirectories(modules.values());
 		dirs.forEach(d -> pathList.addAll(findJarsInDir(d)));
 	}
 
+	/**
+	 * Add extension module lib jars to the given path list.  (This only needed in dev mode to find 
+	 * any pre-built extensions that have been installed, since  we already find extension module 
+	 * jars in production mode.)
+	 * 
+	 * @param pathList The list of paths to add to.
+	 * @param modules The modules to get the jars of.
+	 * @param layout the application layout.
+	 */
+	private static void addExtensionJarPaths(List<String> pathList,
+			Map<String, GModule> modules, GhidraApplicationLayout layout) {
+
+		List<ResourceFile> extensionInstallationDirs = layout.getExtensionInstallationDirs();
+		for (GModule module : modules.values()) {
+
+			ResourceFile moduleDir = module.getModuleRoot();
+			if (!FileUtilities.isPathContainedWithin(extensionInstallationDirs, moduleDir)) {
+				continue; // not an extension
+			}
+
+			Collection<ResourceFile> libDirs =
+				ModuleUtilities.getModuleLibDirectories(Set.of(module));
+			if (libDirs.size() != 1) {
+				continue; // assume multiple lib dirs signals a non-built development project
+			}
+
+			// We have one lib dir; the name 'lib' is used for a fully built extension.  Grab all 
+			// jars from the built extensions lib directory.
+			ResourceFile dir = libDirs.iterator().next();
+			if (dir.getName().equals("lib")) {
+				pathList.addAll(findJarsInDir(dir));
+			}
+		}
+	}
+
 	/**
 	 * Add external runtime lib jars to the given path list.  The external jars are discovered by
 	 * parsing the build/libraryDependencies.txt file that results from a prepDev.

Ghidra/Framework/Utility/src/main/java/utilities/util/FileUtilities.java

@@ -850,7 +850,7 @@ public final class FileUtilities {
 	 *
 	 * @param potentialParentFile The file that may be the parent
 	 * @param otherFile The file that may be the child
-	 * @return boolean true if otherFile's path is within potentialParentFile's path.
+	 * @return boolean true if otherFile's path is within potentialParentFile's path
 	 */
 	public static boolean isPathContainedWithin(File potentialParentFile, File otherFile) {
 		try {
@@ -871,6 +871,20 @@ public final class FileUtilities {
 		}
 	}
 
+	/**
+	 * Returns true if any of the given <code>potentialParents</code> is the parent path of or has
+	 * the same path as the given <code>otherFile</code>.
+	 *
+	 * @param potentialParents The files that may be the parent
+	 * @param otherFile The file that may be the child
+	 * @return boolean true if otherFile's path is within any of the potentialParents' paths 
+	 */
+	public static boolean isPathContainedWithin(Collection<ResourceFile> potentialParents,
+			ResourceFile otherFile) {
+
+		return potentialParents.stream().anyMatch(parent -> parent.containsPath(otherFile));
+	}
+
 	/**
 	 * Returns the portion of the second file that trails the full path of the first file.  If
 	 * the paths are the same or unrelated, then null is returned.
@@ -1250,14 +1264,56 @@ public final class FileUtilities {
 	 * @param consumer the consumer of each child in the given directory
 	 * @throws IOException if there is any problem reading the directory contents
 	 */
-	public static void forEachFile(Path path, Consumer<Stream<Path>> consumer) throws IOException {
-
+	public static void forEachFile(Path path, Consumer<Path> consumer) throws IOException {
 		if (!Files.isDirectory(path)) {
 			return;
 		}
 
 		try (Stream<Path> pathStream = Files.list(path)) {
-			consumer.accept(pathStream);
+			pathStream.forEach(consumer);
 		}
 	}
+
+	/**
+	 * A convenience method to list the contents of the given directory path and pass each to the
+	 * given consumer.  If the given path does not represent a directory, nothing will happen.
+	 * 
+	 * @param resourceFile the directory
+	 * @param consumer the consumer of each child in the given directory
+	 */
+	public static void forEachFile(File resourceFile, Consumer<File> consumer) {
+		if (!resourceFile.isDirectory()) {
+			return;
+		}
+
+		File[] files = resourceFile.listFiles();
+		if (files == null) {
+			return;
+		}
+		for (File child : files) {
+			consumer.accept(child);
+		}
+	}
+
+	/**
+	 * A convenience method to list the contents of the given directory path and pass each to the
+	 * given consumer.  If the given path does not represent a directory, nothing will happen.
+	 * 
+	 * @param resourceFile the directory
+	 * @param consumer the consumer of each child in the given directory
+	 */
+	public static void forEachFile(ResourceFile resourceFile, Consumer<ResourceFile> consumer) {
+		if (!resourceFile.isDirectory()) {
+			return;
+		}
+
+		ResourceFile[] files = resourceFile.listFiles();
+		if (files == null) {
+			return;
+		}
+		for (ResourceFile child : files) {
+			consumer.accept(child);
+		}
+	}
+
 }

Ghidra/Framework/Utility/src/main/java/utility/application/ApplicationLayout.java

@@ -109,7 +109,15 @@ public abstract class ApplicationLayout {
 	}
 
 	/**
-	 * Returns the directory where archived application Extensions are stored.
+	 * Returns the directory where archived application Extensions are stored.  This directory may
+	 * contain both zip files and subdirectories.   This directory is only used inside of an
+	 * installation; development mode does not use this directory.   This directory is used to ship 
+	 * pre-built Ghidra extensions as part of a distribution.
+	 * <P>
+	 * This should be at the following location:<br>
+	 * <ul>
+	 * <li><code>{install dir}/Extensions/Ghidra</code></li>
+	 * </ul>
 	 *
 	 * @return the application Extensions archive directory.  Could be null if the
 	 *   {@link ApplicationLayout} does not support application Extensions.
@@ -120,7 +128,13 @@ public abstract class ApplicationLayout {
 	}
 
 	/**
-	 * Returns an {@link List ordered list} of the application Extensions installation directories.
+	 * Returns a prioritized {@link List ordered list} of the application Extensions installation 
+	 * directories.   Typically, the values may be any of the following locations:<br>
+	 * <ul>
+	 * <li><code>[user settings dir]/Extensions</code></li>
+	 * <li><code>[application install dir]/Ghidra/Extensions</code> (Release Mode)</li>
+	 * <li><code>ghidra/Ghidra/Extensions</code> (Development Mode)</li>
+	 * </ul>
 	 *
 	 * @return an {@link List ordered list} of the application Extensions installation directories.
 	 *   Could be empty if the {@link ApplicationLayout} does not support application Extensions.

Ghidra/Framework/Utility/src/main/java/utility/module/ModuleUtilities.java

@@ -92,12 +92,12 @@ public class ModuleUtilities {
 		if (!rootDir.exists() || remainingDepth <= 0) {
 			return moduleRootDirs;
 		}
-		
-		ResourceFile[] subDirs =  rootDir.listFiles(ResourceFile::isDirectory);
+
+		ResourceFile[] subDirs = rootDir.listFiles(ResourceFile::isDirectory);
 		if (subDirs == null) {
 			throw new RuntimeException("Failed to read directory: " + rootDir);
 		}
-		
+
 		for (ResourceFile subDir : subDirs) {
 			if ("build".equals(subDir.getName())) {
 				continue; // ignore all "build" directories
@@ -230,9 +230,9 @@ public class ModuleUtilities {
 	 * @param modules The modules to get the library directories of.
 	 * @return A collection of library directories from the given modules.
 	 */
-	public static Collection<ResourceFile> getModuleLibDirectories(Map<String, GModule> modules) {
+	public static Collection<ResourceFile> getModuleLibDirectories(Collection<GModule> modules) {
 		List<ResourceFile> libraryDirectories = new ArrayList<>();
-		for (GModule module : modules.values()) {
+		for (GModule module : modules) {
 			module.collectExistingModuleDirs(libraryDirectories, "lib");
 			module.collectExistingModuleDirs(libraryDirectories, "libs");
 		}
@@ -245,10 +245,10 @@ public class ModuleUtilities {
 	 * @param modules The modules to get the compiled .class and resources directories of.
 	 * @return A collection of directories containing classes and resources from the given modules.
 	 */
-	public static Collection<ResourceFile> getModuleBinDirectories(Map<String, GModule> modules) {
+	public static Collection<ResourceFile> getModuleBinDirectories(Collection<GModule> modules) {
 		String[] binaryPathTokens = BINARY_PATH.split(":");
 		List<ResourceFile> binDirectories = new ArrayList<>();
-		for (GModule module : modules.values()) {
+		for (GModule module : modules) {
 			Arrays.stream(binaryPathTokens)
 					.forEach(token -> module.collectExistingModuleDirs(binDirectories, token));
 		}
@@ -404,4 +404,31 @@ public class ModuleUtilities {
 				.map(dir -> dir.getParentFile().getFile(false))
 				.anyMatch(dir -> FileUtilities.isPathContainedWithin(dir, moduleRootDir));
 	}
+
+	/**
+	 * Returns true if the given module has been uninstalled.  
+	 * @param path the module path to check
+	 * @return true if uninstalled
+	 */
+	public static boolean isUninstalled(String path) {
+		return isUninstalled(new File(path));
+	}
+
+	/**
+	 * Returns true if the given module has been uninstalled.  
+	 * @param dir the module dir to check
+	 * @return true if uninstalled
+	 */
+	public static boolean isUninstalled(File dir) {
+		return new File(dir, MANIFEST_FILE_NAME_UNINSTALLED).exists();
+	}
+
+	/**
+	 * Returns true if the given module has been uninstalled.  
+	 * @param dir the module dir to check
+	 * @return true if uninstalled
+	 */
+	public static boolean isUninstalled(ResourceFile dir) {
+		return new ResourceFile(dir, MANIFEST_FILE_NAME_UNINSTALLED).exists();
+	}
 }