Skip to content

Ghidra is a software reverse engineering (SRE) framework

GP-5640: Making more things Iterable

Ryan Kurtz71ed695eddfb
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
index 34d8aecb10..95ee68878b 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/FSUtilities.java
@@ -34,9 +34,7 @@ import docking.widgets.OptionDialog;
import ghidra.app.util.bin.ByteProvider;
import ghidra.formats.gfilesystem.annotations.FileSystemInfo;
import ghidra.formats.gfilesystem.fileinfo.FileType;
-import ghidra.util.HashUtilities;
-import ghidra.util.Msg;
-import ghidra.util.NumericUtilities;
+import ghidra.util.*;
import ghidra.util.exception.CancelledException;
import ghidra.util.exception.CryptoException;
import ghidra.util.task.TaskMonitor;
@@ -204,7 +202,9 @@ public class FSUtilities {
* @return {@link List} of accumulated {@code result}s
* @throws IOException if io error during listing of directories
* @throws CancelledException if user cancels
+ * @deprecated Use {@link GFileSystem#files(GFile)} instead
*/
+ @Deprecated(forRemoval = true, since = "11.4")
public static List<GFile> listFileSystem(GFileSystem fs, GFile dir, List<GFile> result,
TaskMonitor taskMonitor) throws IOException, CancelledException {
if (result == null) {
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
index 07976776b7..cb750ce60e 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystem.java
@@ -16,8 +16,8 @@
package ghidra.formats.gfilesystem;
import java.io.*;
-import java.util.Comparator;
-import java.util.List;
+import java.util.*;
+import java.util.function.Predicate;
import ghidra.app.util.bin.ByteProvider;
import ghidra.app.util.bin.ByteProviderInputStream;
@@ -46,7 +46,7 @@ import ghidra.util.task.TaskMonitor;
* implementations, and usage is being migrated to this interface where possible and as
* time permits.
*/
-public interface GFileSystem extends Closeable, ExtensionPoint {
+public interface GFileSystem extends Closeable, Iterable<GFile>, ExtensionPoint {
/**
* File system volume name.
* <p>
@@ -254,4 +254,52 @@ public interface GFileSystem extends Closeable, ExtensionPoint {
}
+ /**
+ * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+ *
+ * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+ */
+ default Iterable<GFile> files() {
+ return () -> new GFileSystemIterator(this);
+ }
+
+ /**
+ * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+ *
+ * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem}
+ * @throws UncheckedIOException if {@code dir} is not a directory
+ * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+ */
+ default Iterable<GFile> files(GFile dir) throws UncheckedIOException {
+ return () -> new GFileSystemIterator(dir);
+ }
+
+ /**
+ * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+ *
+ * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+ * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+ */
+ default Iterable<GFile> files(Predicate<GFile> fileFilter) {
+ return () -> new GFileSystemIterator(getRootDir(), fileFilter);
+ }
+
+ /**
+ * Gets an {@link Iterator} over this {@link GFileSystem}'s {@link GFile files}.
+ *
+ * @param dir The {@link GFile directory} to start iterating at in this {@link GFileSystem}
+ * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+ * @throws UncheckedIOException if {@code dir} is not a directory
+ * @return An {@link Iterable} over this {@link GFileSystem}'s {@link GFile files}.
+ */
+ default Iterable<GFile> files(GFile dir, Predicate<GFile> fileFilter)
+ throws UncheckedIOException {
+ return () -> new GFileSystemIterator(dir, fileFilter);
+ }
+
+ @Override
+ public default Iterator<GFile> iterator() {
+ return new GFileSystemIterator(this);
+ }
+
}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java
new file mode 100644
index 0000000000..30fe8deedf
--- /dev/null
+++ b/Ghidra/Features/Base/src/main/java/ghidra/formats/gfilesystem/GFileSystemIterator.java
@@ -0,0 +1,110 @@
+/* ###
+ * IP: GHIDRA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ghidra.formats.gfilesystem;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.*;
+import java.util.function.Predicate;
+
+/**
+ * Iterates over the {@link GFile}s in a {@link GFileSystem} depth-first
+ */
+public class GFileSystemIterator implements Iterator<GFile> {
+
+ private Deque<GFile> fileDeque = new ArrayDeque<>();
+ private Deque<GFile> dirDeque = new ArrayDeque<>();
+ private Predicate<GFile> filter;
+
+ /**
+ * Creates a new {@link GFileSystemIterator} at the root of the given {@link GFileSystem}
+ *
+ * @param fs The {@link GFileSystem} to iterate over
+ */
+ public GFileSystemIterator(GFileSystem fs) {
+ this(fs.getRootDir());
+ }
+
+ /**
+ * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory}
+ *
+ * @param dir The {@link GFile directory} to start the iteration at
+ * @throws UncheckedIOException if {@code dir} is not a directory
+ */
+ public GFileSystemIterator(GFile dir) throws UncheckedIOException {
+ this(dir, file -> true);
+ }
+
+ /**
+ * Creates a new {@link GFileSystemIterator} at the given {@link GFile directory}
+ *
+ * @param dir The {@link GFile directory} to start the iteration at
+ * @param fileFilter A filter to apply to the {@link GFile files} iterated over
+ * @throws UncheckedIOException if {@code dir} is not a directory
+ */
+ public GFileSystemIterator(GFile dir, Predicate<GFile> fileFilter) throws UncheckedIOException {
+ if (!dir.isDirectory()) {
+ throw new UncheckedIOException(new IOException("Invalid starting directory!"));
+ }
+ this.dirDeque.push(dir);
+ this.filter = fileFilter;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws UncheckedIOException if an IO-related error occurred on
+ * {@link GFileSystem#getListing(GFile)}
+ */
+ @Override
+ public boolean hasNext() {
+ queueNextFiles();
+ return !fileDeque.isEmpty();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * @throws NoSuchElementException if the iteration has no more elements
+ * @throws UncheckedIOException if an IO-related error occurred on
+ * {@link GFileSystem#getListing(GFile)}
+ */
+ @Override
+ public GFile next() {
+ queueNextFiles();
+ return fileDeque.pop();
+ }
+
+ private void queueNextFiles() throws UncheckedIOException {
+ while (fileDeque.isEmpty() && (!dirDeque.isEmpty())) {
+ try {
+ List<GFile> listing = dirDeque.pop().getListing();
+ listing.stream()
+ .filter(GFile::isDirectory)
+ .sorted(Comparator.comparing(GFile::getName).reversed())
+ .forEach(dirDeque::push);
+ listing.stream()
+ .filter(Predicate.not(GFile::isDirectory))
+ .filter(filter)
+ .sorted(Comparator.comparing(GFile::getName).reversed())
+ .forEach(fileDeque::push);
+ }
+ catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+}
diff --git a/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java b/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java
index d6f57ebd37..bc1d9c1dd0 100644
--- a/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java
+++ b/Ghidra/Features/Base/src/main/java/ghidra/plugins/importer/batch/BatchInfo.java
@@ -335,8 +335,7 @@ public class BatchInfo {
private void processFS(GFileSystem fs, GFile startDir, TaskMonitor taskMonitor)
throws CancelledException, IOException {
- // TODO: drop FSUtils.listFileSystem and do recursion here.
- for (GFile file : FSUtilities.listFileSystem(fs, startDir, null, taskMonitor)) {
+ for (GFile file : fs.files(startDir)) {
taskMonitor.checkCancelled();
FSRL fqFSRL;
try {
diff --git a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java
index 6269819f2b..7b14bfcd32 100644
--- a/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java
+++ b/Ghidra/Features/FileFormats/src/main/java/ghidra/file/jad/JarDecompiler.java
@@ -4,9 +4,9 @@
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,7 +17,6 @@ package ghidra.file.jad;
import java.io.*;
import java.util.Iterator;
-import java.util.List;
import java.util.zip.ZipException;
import org.apache.commons.io.FileUtils;
@@ -94,9 +93,9 @@ public class JarDecompiler {
FileSystemService fsService = FileSystemService.getInstance();
try (GFileSystem fs = fsService.openFileSystemContainer(jarFile, monitor)) {
- List<GFile> files = FSUtilities.listFileSystem(fs, null, null, monitor);
- monitor.initialize(files.size());
- for (GFile file : files) {
+ monitor.setIndeterminate(true);
+ monitor.initialize(0);
+ for (GFile file : fs) {
if (monitor.isCancelled()) {
break;
}
diff --git a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java
index 18971f0c7b..f25900cf61 100644
--- a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java
+++ b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/Project.java
@@ -17,6 +17,7 @@ package ghidra.framework.model;
import java.io.IOException;
import java.net.URL;
+import java.util.Iterator;
import java.util.List;
import ghidra.framework.client.RepositoryAdapter;
@@ -29,7 +30,7 @@ import ghidra.framework.options.SaveState;
* and tools to work together.
*
*/
-public interface Project extends AutoCloseable {
+public interface Project extends AutoCloseable, Iterable<DomainFile> {
/**
* Convenience method to get the name of this project.
@@ -194,4 +195,9 @@ public interface Project extends AutoCloseable {
*/
public void removeProjectViewListener(ProjectViewListener listener);
+ @Override
+ public default Iterator<DomainFile> iterator() {
+ return new ProjectDataUtils.DomainFileIterator(this);
+ }
+
}
diff --git a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java
index 6ba0485c09..767820a6a3 100644
--- a/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java
+++ b/Ghidra/Framework/Project/src/main/java/ghidra/framework/model/ProjectData.java
@@ -17,6 +17,7 @@ package ghidra.framework.model;
import java.io.IOException;
import java.net.URL;
+import java.util.Iterator;
import java.util.List;
import ghidra.framework.client.RepositoryAdapter;
@@ -30,7 +31,7 @@ import ghidra.util.task.TaskMonitor;
* The ProjectData interface provides access to all the data files and folders
* in a project.
*/
-public interface ProjectData {
+public interface ProjectData extends Iterable<DomainFile> {
/**
* @return local storage implementation class
@@ -224,4 +225,8 @@ public interface ProjectData {
*/
public URL getLocalProjectURL();
+ @Override
+ public default Iterator<DomainFile> iterator() {
+ return new ProjectDataUtils.DomainFileIterator(getRootFolder());
+ }
}