From 71ed695eddfb9a75d30a4bd2582b0781c29cf5b4 Mon Sep 17 00:00:00 2001 From: Ryan Kurtz Date: Thu, 8 May 2025 11:42:30 -0400 Subject: [PATCH] GP-5640: Making more things Iterable --- .../formats/gfilesystem/FSUtilities.java | 6 +- .../formats/gfilesystem/GFileSystem.java | 54 ++++++++- .../gfilesystem/GFileSystemIterator.java | 110 ++++++++++++++++++ .../plugins/importer/batch/BatchInfo.java | 3 +- .../java/ghidra/file/jad/JarDecompiler.java | 11 +- .../java/ghidra/framework/model/Project.java | 8 +- .../ghidra/framework/model/ProjectData.java | 7 +- 7 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java index 34d8aecb10..95ee68878b 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java @@ -34,9 +34,7 @@ import docking.widgets.OptionDialog; import ghidra.app.util.bin.ByteProvider; import ghidra.formats.gfilesystem.annotations.FileSystemInfo; import ghidra.formats.gfilesystem.fileinfo.FileType; -import ghidra.util.HashUtilities; -import ghidra.util.Msg; -import ghidra.util.NumericUtilities; +import ghidra.util.*; import ghidra.util.exception.CancelledException; import ghidra.util.exception.CryptoException; import ghidra.util.task.TaskMonitor; @@ -204,7 +202,9 @@ public class FSUtilities { * @return {@link List} of accumulated {@code result}s * @throws IOException if io error during listing of directories * @throws CancelledException if user cancels + * @deprecated Use {@link GFileSystem#files(GFile)} instead */ + @Deprecated(forRemoval = true, since = "11.4") public static List listFileSystem(GFileSystem fs, GFile dir, List result, TaskMonitor taskMonitor) throws IOException, CancelledException { if (result == null) { diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java index 07976776b7..cb750ce60e 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java @@ -16,8 +16,8 @@ package ghidra.formats.gfilesystem; import java.io.*; -import java.util.Comparator; -import java.util.List; +import java.util.*; +import java.util.function.Predicate; import ghidra.app.util.bin.ByteProvider; import ghidra.app.util.bin.ByteProviderInputStream; @@ -46,7 +46,7 @@ import ghidra.util.task.TaskMonitor; * implementations, and usage is being migrated to this interface where possible and as * time permits. */ -public interface GFileSystem extends Closeable, ExtensionPoint { +public interface GFileSystem extends Closeable, Iterable, ExtensionPoint { /** * File system volume name. *

@@ -254,4 +254,52 @@ public interface GFileSystem extends Closeable, ExtensionPoint { } + /** + * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}. + * + * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}. + */ + default Iterable files() { + return () -> new GFileSystemIterator(this); + } + + /** + * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}. + * + * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem} + * @throws UncheckedIOException if {@code dir} is not a directory + * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}. + */ + default Iterable files(GFile dir) throws UncheckedIOException { + return () -> new GFileSystemIterator(dir); + } + + /** + * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}. + * + * @param fileFilter A filter to apply to the {@link GFile files} iterated over + * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}. + */ + default Iterable files(Predicate fileFilter) { + return () -> new GFileSystemIterator(getRootDir(), fileFilter); + } + + /** + * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}. + * + * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem} + * @param fileFilter A filter to apply to the {@link GFile files} iterated over + * @throws UncheckedIOException if {@code dir} is not a directory + * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}. + */ + default Iterable files(GFile dir, Predicate fileFilter) + throws UncheckedIOException { + return () -> new GFileSystemIterator(dir, fileFilter); + } + + @Override + public default Iterator iterator() { + return new GFileSystemIterator(this); + } + } diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java new file mode 100644 index 0000000000..30fe8deedf --- /dev/null +++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java @@ -0,0 +1,110 @@ +/* ### + * IP: GHIDRA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ghidra.formats.gfilesystem; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.*; +import java.util.function.Predicate; + +/** + * Iterates over the {@link GFile}s in a {@link GFileSystem} depth-first + */ +public class GFileSystemIterator implements Iterator { + + private Deque fileDeque = new ArrayDeque<>(); + private Deque dirDeque = new ArrayDeque<>(); + private Predicate filter; + + /** + * Creates a new {@link GFileSystemIterator} at the root of the given {@link GFileSystem} + * + * @param fs The {@link GFileSystem} to iterate over + */ + public GFileSystemIterator(GFileSystem fs) { + this(fs.getRootDir()); + } + + /** + * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory} + * + * @param dir The {@link GFile directory} to start the iteration at + * @throws UncheckedIOException if {@code dir} is not a directory + */ + public GFileSystemIterator(GFile dir) throws UncheckedIOException { + this(dir, file -> true); + } + + /** + * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory} + * + * @param dir The {@link GFile directory} to start the iteration at + * @param fileFilter A filter to apply to the {@link GFile files} iterated over + * @throws UncheckedIOException if {@code dir} is not a directory + */ + public GFileSystemIterator(GFile dir, Predicate fileFilter) throws UncheckedIOException { + if (!dir.isDirectory()) { + throw new UncheckedIOException(new IOException("Invalid starting directory!")); + } + this.dirDeque.push(dir); + this.filter = fileFilter; + } + + /** + * {@inheritDoc} + * + * @throws UncheckedIOException if an IO-related error occurred on + * {@link GFileSystem#getListing(GFile)} + */ + @Override + public boolean hasNext() { + queueNextFiles(); + return !fileDeque.isEmpty(); + } + + /** + * {@inheritDoc} + * + * @throws NoSuchElementException if the iteration has no more elements + * @throws UncheckedIOException if an IO-related error occurred on + * {@link GFileSystem#getListing(GFile)} + */ + @Override + public GFile next() { + queueNextFiles(); + return fileDeque.pop(); + } + + private void queueNextFiles() throws UncheckedIOException { + while (fileDeque.isEmpty() && (!dirDeque.isEmpty())) { + try { + List listing = dirDeque.pop().getListing(); + listing.stream() + .filter(GFile::isDirectory) + .sorted(Comparator.comparing(GFile::getName).reversed()) + .forEach(dirDeque::push); + listing.stream() + .filter(Predicate.not(GFile::isDirectory)) + .filter(filter) + .sorted(Comparator.comparing(GFile::getName).reversed()) + .forEach(fileDeque::push); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } +} diff --git a/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java b/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java index d6f57ebd37..bc1d9c1dd0 100644 --- a/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java +++ b/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java @@ -335,8 +335,7 @@ public class BatchInfo { private void processFS(GFileSystem fs, GFile startDir, TaskMonitor taskMonitor) throws CancelledException, IOException { - // TODO: drop FSUtils.listFileSystem and do recursion here. - for (GFile file : FSUtilities.listFileSystem(fs, startDir, null, taskMonitor)) { + for (GFile file : fs.files(startDir)) { taskMonitor.checkCancelled(); FSRL fqFSRL; try { diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java index 6269819f2b..7b14bfcd32 100644 --- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java +++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,6 @@ package ghidra.file.jad; import java.io.*; import java.util.Iterator; -import java.util.List; import java.util.zip.ZipException; import org.apache.commons.io.FileUtils; @@ -94,9 +93,9 @@ public class JarDecompiler { FileSystemService fsService = FileSystemService.getInstance(); try (GFileSystem fs = fsService.openFileSystemContainer(jarFile, monitor)) { - List files = FSUtilities.listFileSystem(fs, null, null, monitor); - monitor.initialize(files.size()); - for (GFile file : files) { + monitor.setIndeterminate(true); + monitor.initialize(0); + for (GFile file : fs) { if (monitor.isCancelled()) { break; } diff --git a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java index 18971f0c7b..f25900cf61 100644 --- a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java +++ b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java @@ -17,6 +17,7 @@ package ghidra.framework.model; import java.io.IOException; import java.net.URL; +import java.util.Iterator; import java.util.List; import ghidra.framework.client.RepositoryAdapter; @@ -29,7 +30,7 @@ import ghidra.framework.options.SaveState; * and tools to work together. * */ -public interface Project extends AutoCloseable { +public interface Project extends AutoCloseable, Iterable { /** * Convenience method to get the name of this project. @@ -194,4 +195,9 @@ public interface Project extends AutoCloseable { */ public void removeProjectViewListener(ProjectViewListener listener); + @Override + public default Iterator iterator() { + return new ProjectDataUtils.DomainFileIterator(this); + } + } diff --git a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java index 6ba0485c09..767820a6a3 100644 --- a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java +++ b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java @@ -17,6 +17,7 @@ package ghidra.framework.model; import java.io.IOException; import java.net.URL; +import java.util.Iterator; import java.util.List; import ghidra.framework.client.RepositoryAdapter; @@ -30,7 +31,7 @@ import ghidra.util.task.TaskMonitor; * The ProjectData interface provides access to all the data files and folders * in a project. */ -public interface ProjectData { +public interface ProjectData extends Iterable { /** * @return local storage implementation class @@ -224,4 +225,8 @@ public interface ProjectData { */ public URL getLocalProjectURL(); + @Override + public default Iterator iterator() { + return new ProjectDataUtils.DomainFileIterator(getRootFolder()); + } }