Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
refactor: extract FileDiscovery with git ls-files support
riley0227934ad7a847f2
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalaindex 41e520eb..44c93dcb 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -111,65 +111,13 @@ class BulkIngestionService(language: Option[String],recursive: Boolean): IO[List[Path]] = IO.blocking {- import scala.jdk.CollectionConverters._-- val extensions = language match {- case Some("python") => Set(".py")- case Some("typescript") => Set(".ts", ".tsx")- case Some("scala") => Set(".scala", ".sc")- case Some("config") => Set(".json", ".yaml", ".yml", ".toml")- case Some("markdown") => Set(".md")- case _ => Set(- ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",- ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",- ".kt", ".kts", ".swift", ".rb", ".php",- ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",- ".md", ".mdx", ".rst", ".txt"- )- }-- val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")-+ val extensions = FileDiscovery.extensionsFor(language)if (Files.isRegularFile(path)) {- val base = path.getFileName.toString- if (specialFiles.contains(base) || extensions.exists(base.endsWith)) List(path)- else List.empty+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty} else if (Files.isDirectory(path)) {- if (recursive) {- val ignoredDirs = Set(- "node_modules", ".git", "target", "dist", "build", ".next",- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",- ".gradle", ".idea", ".vscode", ".settings", "out",- ".cache", ".parcel-cache", "coverage", ".nyc_output"- )- val result = scala.collection.mutable.ListBuffer.empty[Path]- Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {- override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {- if (dir != path && ignoredDirs.contains(dir.getFileName.toString))- java.nio.file.FileVisitResult.SKIP_SUBTREE- else- java.nio.file.FileVisitResult.CONTINUE- }- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {- val base = file.getFileName.toString- if (specialFiles.contains(base) || extensions.exists(base.endsWith))- result += file- java.nio.file.FileVisitResult.CONTINUE- }- })- result.toList- } else {- val stream = Files.list(path)- try {- stream.iterator().asScala- .filter(Files.isRegularFile(_))- .filter { p =>- val base = p.getFileName.toString- specialFiles.contains(base) || extensions.exists(base.endsWith)- }- .toList- } finally stream.close()- }+ // Try git ls-files first (respects .gitignore), fall back to manual walk+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))} else List.empty}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scalanew file mode 100644index 00000000..2c139642--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala@@ -0,0 +1,133 @@+package ix.memory.ingestion++import java.nio.file.{Files, Path}++/**+ * Discovers source files for ingestion.+ *+ * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.+ * Fallback: manual file walk with hardcoded ignore list (for non-git directories).+ */+object FileDiscovery {++ private val defaultExtensions = Set(+ // code+ ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",+ ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",+ ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".ex", ".exs",+ ".lua", ".zig", ".nim", ".jl", ".r", ".R", ".pl", ".pm",+ ".sh", ".bash", ".zsh", ".fish",++ // config / data+ ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",+ ".xml", ".plist", ".gradle",++ // docs+ ".md", ".mdx", ".rst", ".txt",++ // build+ ".sbt", ".gradle", ".cmake"+ )++ /** Files matched by name regardless of extension. */+ private val specialFiles = Set(+ "Dockerfile", "Makefile", "Rakefile", "Gemfile", "Procfile",+ "README", "LICENSE", "NOTICE", "CHANGELOG",+ "Cargo.toml", "go.mod", "go.sum",+ "package.json", "tsconfig.json", "pyproject.toml", "setup.py", "setup.cfg",+ "build.sbt", "build.gradle", "pom.xml", "CMakeLists.txt",+ ".gitignore", ".dockerignore", ".editorconfig"+ )++ def extensionsFor(language: Option[String]): Set[String] = language match {+ case Some("python") => Set(".py")+ case Some("typescript") => Set(".ts", ".tsx")+ case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")+ case Some("scala") => Set(".scala", ".sc")+ case Some("java") => Set(".java")+ case Some("go") => Set(".go")+ case Some("rust") => Set(".rs")+ case Some("ruby") => Set(".rb")+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")+ case Some("markdown") => Set(".md", ".mdx")+ case _ => defaultExtensions+ }++ def matchesFilter(file: Path, extensions: Set[String]): Boolean = {+ val name = file.getFileName.toString+ specialFiles.contains(name) || extensions.exists(name.endsWith)+ }++ /**+ * Use `git ls-files` to discover tracked files. Returns None if not a git repo+ * or git is not available.+ */+ def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {+ try {+ val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")+ pb.directory(dir.toFile)+ pb.redirectErrorStream(true)+ val proc = pb.start()+ val output = new String(proc.getInputStream.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)+ val exitCode = proc.waitFor()+ if (exitCode != 0) return None++ val files = output.split("\n").toList+ .filter(_.nonEmpty)+ .map(rel => dir.resolve(rel).normalize())+ .filter(p => Files.isRegularFile(p))+ .filter { p =>+ val isUnder = if (recursive) true else p.getParent == dir+ isUnder && matchesFilter(p, extensions)+ }++ Some(files)+ } catch {+ case _: Throwable => None+ }+ }++ /**+ * Fallback: walk the directory tree, skipping common non-source directories.+ */+ def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {+ import scala.jdk.CollectionConverters._++ if (!recursive) {+ val stream = Files.list(dir)+ try {+ stream.iterator().asScala+ .filter(Files.isRegularFile(_))+ .filter(matchesFilter(_, extensions))+ .toList+ } finally stream.close()+ } else {+ // Directories to skip when not using git+ val ignoredDirs = Set(+ "node_modules", ".git", ".hg", ".svn",+ "target", "dist", "build", "out", ".next",+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",+ ".gradle", ".idea", ".vscode", ".settings", ".vs",+ ".cache", ".parcel-cache", "coverage", ".nyc_output",+ "vendor", "Pods", ".dart_tool", ".pub-cache",+ "bin", "obj", "pkg"+ )+ val result = scala.collection.mutable.ListBuffer.empty[Path]+ Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {+ override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ val name = d.getFileName.toString+ if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))+ java.nio.file.FileVisitResult.SKIP_SUBTREE+ else+ java.nio.file.FileVisitResult.CONTINUE+ }+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ if (matchesFilter(file, extensions))+ result += file+ java.nio.file.FileVisitResult.CONTINUE+ }+ })+ result.toList+ }+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 920b6854..710fff1c 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -98,79 +98,12 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, querlanguage: Option[String],recursive: Boolean): IO[List[Path]] = IO.blocking {- import scala.jdk.CollectionConverters._-- val extensions = language match {- case Some("python") => Set(".py")- case Some("typescript") => Set(".ts", ".tsx")- case Some("scala") => Set(".scala", ".sc")- case Some("config") => Set(".json", ".yaml", ".yml", ".toml")- case Some("markdown") => Set(".md")- case _ => Set(- // code- ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",- ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",- ".kt", ".kts", ".swift", ".rb", ".php",-- // config / data- ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",-- // docs- ".md", ".mdx", ".rst", ".txt"- )- }-+ val extensions = FileDiscovery.extensionsFor(language)if (Files.isRegularFile(path)) {- val base = path.getFileName.toString- val isSpecial = Set(- "Dockerfile",- "Makefile",- "README",- "LICENSE",- "NOTICE",- "build.sbt"- ).contains(base)- if (isSpecial || extensions.exists(ext => base.endsWith(ext))) List(path)- else List.empty+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty} else if (Files.isDirectory(path)) {- val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")- if (recursive) {- val ignoredDirs = Set(- "node_modules", ".git", "target", "dist", "build", ".next",- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",- ".gradle", ".idea", ".vscode", ".settings", "out",- ".cache", ".parcel-cache", "coverage", ".nyc_output"- )- val result = scala.collection.mutable.ListBuffer.empty[Path]- Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {- override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {- if (dir != path && ignoredDirs.contains(dir.getFileName.toString))- java.nio.file.FileVisitResult.SKIP_SUBTREE- else- java.nio.file.FileVisitResult.CONTINUE- }- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {- val base = file.getFileName.toString- if (specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext)))- result += file- java.nio.file.FileVisitResult.CONTINUE- }- })- result.toList- } else {- val stream = Files.list(path)- try {- stream.iterator().asScala- .filter(p => Files.isRegularFile(p))- .filter { p =>- val base = p.getFileName.toString- specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext))- }- .toList- } finally {- stream.close()- }- }+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))} else {List.empty}