Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

refactor: extract FileDiscovery with git ls-files support

riley0227934ad7a847f2
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index 41e520eb..44c93dcb 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -111,65 +111,13 @@ class BulkIngestionService(
language: Option[String],
recursive: Boolean
): IO[List[Path]] = IO.blocking {
- import scala.jdk.CollectionConverters._
-
- val extensions = language match {
- case Some("python") => Set(".py")
- case Some("typescript") => Set(".ts", ".tsx")
- case Some("scala") => Set(".scala", ".sc")
- case Some("config") => Set(".json", ".yaml", ".yml", ".toml")
- case Some("markdown") => Set(".md")
- case _ => Set(
- ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
- ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",
- ".kt", ".kts", ".swift", ".rb", ".php",
- ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",
- ".md", ".mdx", ".rst", ".txt"
- )
- }
-
- val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")
-
+ val extensions = FileDiscovery.extensionsFor(language)
if (Files.isRegularFile(path)) {
- val base = path.getFileName.toString
- if (specialFiles.contains(base) || extensions.exists(base.endsWith)) List(path)
- else List.empty
+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
} else if (Files.isDirectory(path)) {
- if (recursive) {
- val ignoredDirs = Set(
- "node_modules", ".git", "target", "dist", "build", ".next",
- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",
- ".gradle", ".idea", ".vscode", ".settings", "out",
- ".cache", ".parcel-cache", "coverage", ".nyc_output"
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
- Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {
- override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- if (dir != path && ignoredDirs.contains(dir.getFileName.toString))
- java.nio.file.FileVisitResult.SKIP_SUBTREE
- else
- java.nio.file.FileVisitResult.CONTINUE
- }
- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- val base = file.getFileName.toString
- if (specialFiles.contains(base) || extensions.exists(base.endsWith))
- result += file
- java.nio.file.FileVisitResult.CONTINUE
- }
- })
- result.toList
- } else {
- val stream = Files.list(path)
- try {
- stream.iterator().asScala
- .filter(Files.isRegularFile(_))
- .filter { p =>
- val base = p.getFileName.toString
- specialFiles.contains(base) || extensions.exists(base.endsWith)
- }
- .toList
- } finally stream.close()
- }
+ // Try git ls-files first (respects .gitignore), fall back to manual walk
+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)
+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
} else List.empty
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
new file mode 100644
index 00000000..2c139642
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
@@ -0,0 +1,133 @@
+package ix.memory.ingestion
+
+import java.nio.file.{Files, Path}
+
+/**
+ * Discovers source files for ingestion.
+ *
+ * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.
+ * Fallback: manual file walk with hardcoded ignore list (for non-git directories).
+ */
+object FileDiscovery {
+
+ private val defaultExtensions = Set(
+ // code
+ ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+ ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",
+ ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".ex", ".exs",
+ ".lua", ".zig", ".nim", ".jl", ".r", ".R", ".pl", ".pm",
+ ".sh", ".bash", ".zsh", ".fish",
+
+ // config / data
+ ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",
+ ".xml", ".plist", ".gradle",
+
+ // docs
+ ".md", ".mdx", ".rst", ".txt",
+
+ // build
+ ".sbt", ".gradle", ".cmake"
+ )
+
+ /** Files matched by name regardless of extension. */
+ private val specialFiles = Set(
+ "Dockerfile", "Makefile", "Rakefile", "Gemfile", "Procfile",
+ "README", "LICENSE", "NOTICE", "CHANGELOG",
+ "Cargo.toml", "go.mod", "go.sum",
+ "package.json", "tsconfig.json", "pyproject.toml", "setup.py", "setup.cfg",
+ "build.sbt", "build.gradle", "pom.xml", "CMakeLists.txt",
+ ".gitignore", ".dockerignore", ".editorconfig"
+ )
+
+ def extensionsFor(language: Option[String]): Set[String] = language match {
+ case Some("python") => Set(".py")
+ case Some("typescript") => Set(".ts", ".tsx")
+ case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")
+ case Some("scala") => Set(".scala", ".sc")
+ case Some("java") => Set(".java")
+ case Some("go") => Set(".go")
+ case Some("rust") => Set(".rs")
+ case Some("ruby") => Set(".rb")
+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")
+ case Some("markdown") => Set(".md", ".mdx")
+ case _ => defaultExtensions
+ }
+
+ def matchesFilter(file: Path, extensions: Set[String]): Boolean = {
+ val name = file.getFileName.toString
+ specialFiles.contains(name) || extensions.exists(name.endsWith)
+ }
+
+ /**
+ * Use `git ls-files` to discover tracked files. Returns None if not a git repo
+ * or git is not available.
+ */
+ def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {
+ try {
+ val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")
+ pb.directory(dir.toFile)
+ pb.redirectErrorStream(true)
+ val proc = pb.start()
+ val output = new String(proc.getInputStream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)
+ val exitCode = proc.waitFor()
+ if (exitCode != 0) return None
+
+ val files = output.split("\n").toList
+ .filter(_.nonEmpty)
+ .map(rel => dir.resolve(rel).normalize())
+ .filter(p => Files.isRegularFile(p))
+ .filter { p =>
+ val isUnder = if (recursive) true else p.getParent == dir
+ isUnder && matchesFilter(p, extensions)
+ }
+
+ Some(files)
+ } catch {
+ case _: Throwable => None
+ }
+ }
+
+ /**
+ * Fallback: walk the directory tree, skipping common non-source directories.
+ */
+ def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {
+ import scala.jdk.CollectionConverters._
+
+ if (!recursive) {
+ val stream = Files.list(dir)
+ try {
+ stream.iterator().asScala
+ .filter(Files.isRegularFile(_))
+ .filter(matchesFilter(_, extensions))
+ .toList
+ } finally stream.close()
+ } else {
+ // Directories to skip when not using git
+ val ignoredDirs = Set(
+ "node_modules", ".git", ".hg", ".svn",
+ "target", "dist", "build", "out", ".next",
+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
+ ".gradle", ".idea", ".vscode", ".settings", ".vs",
+ ".cache", ".parcel-cache", "coverage", ".nyc_output",
+ "vendor", "Pods", ".dart_tool", ".pub-cache",
+ "bin", "obj", "pkg"
+ )
+ val result = scala.collection.mutable.ListBuffer.empty[Path]
+ Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {
+ override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ val name = d.getFileName.toString
+ if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))
+ java.nio.file.FileVisitResult.SKIP_SUBTREE
+ else
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ if (matchesFilter(file, extensions))
+ result += file
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ })
+ result.toList
+ }
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 920b6854..710fff1c 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -98,79 +98,12 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
language: Option[String],
recursive: Boolean
): IO[List[Path]] = IO.blocking {
- import scala.jdk.CollectionConverters._
-
- val extensions = language match {
- case Some("python") => Set(".py")
- case Some("typescript") => Set(".ts", ".tsx")
- case Some("scala") => Set(".scala", ".sc")
- case Some("config") => Set(".json", ".yaml", ".yml", ".toml")
- case Some("markdown") => Set(".md")
- case _ => Set(
- // code
- ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
- ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",
- ".kt", ".kts", ".swift", ".rb", ".php",
-
- // config / data
- ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",
-
- // docs
- ".md", ".mdx", ".rst", ".txt"
- )
- }
-
+ val extensions = FileDiscovery.extensionsFor(language)
if (Files.isRegularFile(path)) {
- val base = path.getFileName.toString
- val isSpecial = Set(
- "Dockerfile",
- "Makefile",
- "README",
- "LICENSE",
- "NOTICE",
- "build.sbt"
- ).contains(base)
- if (isSpecial || extensions.exists(ext => base.endsWith(ext))) List(path)
- else List.empty
+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
} else if (Files.isDirectory(path)) {
- val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")
- if (recursive) {
- val ignoredDirs = Set(
- "node_modules", ".git", "target", "dist", "build", ".next",
- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",
- ".gradle", ".idea", ".vscode", ".settings", "out",
- ".cache", ".parcel-cache", "coverage", ".nyc_output"
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
- Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {
- override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- if (dir != path && ignoredDirs.contains(dir.getFileName.toString))
- java.nio.file.FileVisitResult.SKIP_SUBTREE
- else
- java.nio.file.FileVisitResult.CONTINUE
- }
- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- val base = file.getFileName.toString
- if (specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext)))
- result += file
- java.nio.file.FileVisitResult.CONTINUE
- }
- })
- result.toList
- } else {
- val stream = Files.list(path)
- try {
- stream.iterator().asScala
- .filter(p => Files.isRegularFile(p))
- .filter { p =>
- val base = p.getFileName.toString
- specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext))
- }
- .toList
- } finally {
- stream.close()
- }
- }
+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)
+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
} else {
List.empty
}