Merge remote-tracking branch 'origin/GP-5640_ryanmkurtz_iterableFS--SQUASHED'

Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java

@@ -34,9 +34,7 @@ import docking.widgets.OptionDialog;
 import ghidra.app.util.bin.ByteProvider;
 import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
 import ghidra.formats.gfilesystem.fileinfo.FileType;
-import ghidra.util.HashUtilities;
+import ghidra.util.*;
-import ghidra.util.Msg;
-import ghidra.util.NumericUtilities;
 import ghidra.util.exception.CancelledException;
 import ghidra.util.exception.CryptoException;
 import ghidra.util.task.TaskMonitor;
@@ -204,7 +202,9 @@ public class FSUtilities {
 	 * @return {@link List} of accumulated {@code result}s
 	 * @throws IOException if io error during listing of directories
 	 * @throws CancelledException if user cancels
+	 * @deprecated Use {@link GFileSystem#files(GFile)} instead
 	 */
+	@Deprecated(forRemoval = true, since = "11.4")
 	public static List<GFile> listFileSystem(GFileSystem fs, GFile dir, List<GFile> result,
 			TaskMonitor taskMonitor) throws IOException, CancelledException {
 		if (result == null) {

Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java

@@ -16,8 +16,8 @@
 package ghidra.formats.gfilesystem;
 
 import java.io.*;
-import java.util.Comparator;
+import java.util.*;
-import java.util.List;
+import java.util.function.Predicate;
 
 import ghidra.app.util.bin.ByteProvider;
 import ghidra.app.util.bin.ByteProviderInputStream;
@@ -46,7 +46,7 @@ import ghidra.util.task.TaskMonitor;
  * implementations, and usage is being migrated to this interface where possible and as
  * time permits.
  */
-public interface GFileSystem extends Closeable, ExtensionPoint {
+public interface GFileSystem extends Closeable, Iterable<GFile>, ExtensionPoint {
 	/**
 	 * File system volume name.
 	 * <p>
@@ -254,4 +254,52 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
 
 	}
 
+	/**
+	 * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+	 * 
+	 * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+	 */
+	default Iterable<GFile> files() {
+		return () -> new GFileSystemIterator(this);
+	}
+
+	/**
+	 * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+	 * 
+	 * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem}
+	 * @throws UncheckedIOException if {@code dir} is not a directory
+	 * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+	 */
+	default Iterable<GFile> files(GFile dir) throws UncheckedIOException {
+		return () -> new GFileSystemIterator(dir);
+	}
+
+	/**
+	 * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+	 * 
+	 * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+	 * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+	 */
+	default Iterable<GFile> files(Predicate<GFile> fileFilter) {
+		return () -> new GFileSystemIterator(getRootDir(), fileFilter);
+	}
+
+	/**
+	 * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+	 * 
+	 * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem}
+	 * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+	 * @throws UncheckedIOException if {@code dir} is not a directory
+	 * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+	 */
+	default Iterable<GFile> files(GFile dir, Predicate<GFile> fileFilter)
+			throws UncheckedIOException {
+		return () -> new GFileSystemIterator(dir, fileFilter);
+	}
+
+	@Override
+	public default Iterator<GFile> iterator() {
+		return new GFileSystemIterator(this);
+	}
+
 }

Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java

@@ -0,0 +1,110 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.formats.gfilesystem;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.*;
+import java.util.function.Predicate;
+
+/**
+ * Iterates over the {@link GFile}s in a {@link GFileSystem} depth-first
+ */
+public class GFileSystemIterator implements Iterator<GFile> {
+
+	private Deque<GFile> fileDeque = new ArrayDeque<>();
+	private Deque<GFile> dirDeque = new ArrayDeque<>();
+	private Predicate<GFile> filter;
+
+	/**
+	 * Creates a new {@link GFileSystemIterator} at the root of the given {@link GFileSystem}
+	 * 
+	 * @param fs The {@link GFileSystem} to iterate over
+	 */
+	public GFileSystemIterator(GFileSystem fs) {
+		this(fs.getRootDir());
+	}
+
+	/**
+	 * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory}
+	 * 
+	 * @param dir The {@link GFile directory} to start the iteration at
+	 * @throws UncheckedIOException if {@code dir} is not a directory
+	 */
+	public GFileSystemIterator(GFile dir) throws UncheckedIOException {
+		this(dir, file -> true);
+	}
+
+	/**
+	 * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory}
+	 * 
+	 * @param dir The {@link GFile directory} to start the iteration at
+	 * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+	 * @throws UncheckedIOException if {@code dir} is not a directory
+	 */
+	public GFileSystemIterator(GFile dir, Predicate<GFile> fileFilter) throws UncheckedIOException {
+		if (!dir.isDirectory()) {
+			throw new UncheckedIOException(new IOException("Invalid starting directory!"));
+		}
+		this.dirDeque.push(dir);
+		this.filter = fileFilter;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @throws UncheckedIOException if an IO-related error occurred on 
+	 *   {@link GFileSystem#getListing(GFile)}
+	 */
+	@Override
+	public boolean hasNext() {
+		queueNextFiles();
+		return !fileDeque.isEmpty();
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @throws NoSuchElementException if the iteration has no more elements
+	 * @throws UncheckedIOException if an IO-related error occurred on 
+	 *   {@link GFileSystem#getListing(GFile)}
+	 */
+	@Override
+	public GFile next() {
+		queueNextFiles();
+		return fileDeque.pop();
+	}
+
+	private void queueNextFiles() throws UncheckedIOException {
+		while (fileDeque.isEmpty() && (!dirDeque.isEmpty())) {
+			try {
+				List<GFile> listing = dirDeque.pop().getListing();
+				listing.stream()
+						.filter(GFile::isDirectory)
+						.sorted(Comparator.comparing(GFile::getName).reversed())
+						.forEach(dirDeque::push);
+				listing.stream()
+						.filter(Predicate.not(GFile::isDirectory))
+						.filter(filter)
+						.sorted(Comparator.comparing(GFile::getName).reversed())
+						.forEach(fileDeque::push);
+			}
+			catch (IOException e) {
+				throw new UncheckedIOException(e);
+			}
+		}
+	}
+}

Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java

@@ -335,8 +335,7 @@ public class BatchInfo {
 	private void processFS(GFileSystem fs, GFile startDir, TaskMonitor taskMonitor)
 			throws CancelledException, IOException {
 
-		// TODO: drop FSUtils.listFileSystem and do recursion here.
+		for (GFile file : fs.files(startDir)) {
-		for (GFile file : FSUtilities.listFileSystem(fs, startDir, null, taskMonitor)) {
 			taskMonitor.checkCancelled();
 			FSRL fqFSRL;
 			try {

Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java

@@ -4,9 +4,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,7 +17,6 @@ package ghidra.file.jad;
 
 import java.io.*;
 import java.util.Iterator;
-import java.util.List;
 import java.util.zip.ZipException;
 
 import org.apache.commons.io.FileUtils;
@@ -94,9 +93,9 @@ public class JarDecompiler {
 
 		FileSystemService fsService = FileSystemService.getInstance();
 		try (GFileSystem fs = fsService.openFileSystemContainer(jarFile, monitor)) {
-			List<GFile> files = FSUtilities.listFileSystem(fs, null, null, monitor);
+			monitor.setIndeterminate(true);
-			monitor.initialize(files.size());
+			monitor.initialize(0);
-			for (GFile file : files) {
+			for (GFile file : fs) {
 				if (monitor.isCancelled()) {
 					break;
 				}

Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java

@@ -17,6 +17,7 @@ package ghidra.framework.model;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.Iterator;
 import java.util.List;
 
 import ghidra.framework.client.RepositoryAdapter;
@@ -29,7 +30,7 @@ import ghidra.framework.options.SaveState;
  * and tools to work together.
  * 
  */
-public interface Project extends AutoCloseable {
+public interface Project extends AutoCloseable, Iterable<DomainFile> {
 
 	/**
 	 * Convenience method to get the name of this project.
@@ -194,4 +195,9 @@ public interface Project extends AutoCloseable {
 	 */
 	public void removeProjectViewListener(ProjectViewListener listener);
 
+	@Override
+	public default Iterator<DomainFile> iterator() {
+		return new ProjectDataUtils.DomainFileIterator(this);
+	}
+
 }

Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java

@@ -17,6 +17,7 @@ package ghidra.framework.model;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.Iterator;
 import java.util.List;
 
 import ghidra.framework.client.RepositoryAdapter;
@@ -30,7 +31,7 @@ import ghidra.util.task.TaskMonitor;
  * The ProjectData interface provides access to all the data files and folders
  * in a project.
  */
-public interface ProjectData {
+public interface ProjectData extends Iterable<DomainFile> {
 
 	/**
 	 * @return local storage implementation class
@@ -224,4 +225,8 @@ public interface ProjectData {
 	 */
 	public URL getLocalProjectURL();
 
+	@Override
+	public default Iterator<DomainFile> iterator() {
+		return new ProjectDataUtils.DomainFileIterator(getRootFolder());
+	}
 }