Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix: skip node_modules, .git, target and other ignored dirs during file discovery

riley02277f56b65eeb69
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index 27a95d21..41e520eb 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -135,16 +135,41 @@ class BulkIngestionService(
if (specialFiles.contains(base) || extensions.exists(base.endsWith)) List(path)
else List.empty
} else if (Files.isDirectory(path)) {
- val stream = if (recursive) Files.walk(path) else Files.list(path)
- try {
- stream.iterator().asScala
- .filter(Files.isRegularFile(_))
- .filter { p =>
- val base = p.getFileName.toString
- specialFiles.contains(base) || extensions.exists(base.endsWith)
+ if (recursive) {
+ val ignoredDirs = Set(
+ "node_modules", ".git", "target", "dist", "build", ".next",
+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",
+ ".gradle", ".idea", ".vscode", ".settings", "out",
+ ".cache", ".parcel-cache", "coverage", ".nyc_output"
+ )
+ val result = scala.collection.mutable.ListBuffer.empty[Path]
+ Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {
+ override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ if (dir != path && ignoredDirs.contains(dir.getFileName.toString))
+ java.nio.file.FileVisitResult.SKIP_SUBTREE
+ else
+ java.nio.file.FileVisitResult.CONTINUE
}
- .toList
- } finally stream.close()
+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ val base = file.getFileName.toString
+ if (specialFiles.contains(base) || extensions.exists(base.endsWith))
+ result += file
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ })
+ result.toList
+ } else {
+ val stream = Files.list(path)
+ try {
+ stream.iterator().asScala
+ .filter(Files.isRegularFile(_))
+ .filter { p =>
+ val base = p.getFileName.toString
+ specialFiles.contains(base) || extensions.exists(base.endsWith)
+ }
+ .toList
+ } finally stream.close()
+ }
} else List.empty
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 7390d36d..920b6854 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -133,26 +133,43 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
if (isSpecial || extensions.exists(ext => base.endsWith(ext))) List(path)
else List.empty
} else if (Files.isDirectory(path)) {
- val stream = if (recursive) Files.walk(path) else Files.list(path)
- try {
- stream.iterator().asScala
- .filter(p => Files.isRegularFile(p))
- .filter { p =>
- val base = p.getFileName.toString
- val isSpecial = Set(
- "Dockerfile",
- "Makefile",
- "README",
- "LICENSE",
- "NOTICE",
- "build.sbt"
- ).contains(base)
- val matchesExt = extensions.exists(ext => base.endsWith(ext))
- matchesExt || isSpecial
+ val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")
+ if (recursive) {
+ val ignoredDirs = Set(
+ "node_modules", ".git", "target", "dist", "build", ".next",
+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",
+ ".gradle", ".idea", ".vscode", ".settings", "out",
+ ".cache", ".parcel-cache", "coverage", ".nyc_output"
+ )
+ val result = scala.collection.mutable.ListBuffer.empty[Path]
+ Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {
+ override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ if (dir != path && ignoredDirs.contains(dir.getFileName.toString))
+ java.nio.file.FileVisitResult.SKIP_SUBTREE
+ else
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ val base = file.getFileName.toString
+ if (specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext)))
+ result += file
+ java.nio.file.FileVisitResult.CONTINUE
}
- .toList
- } finally {
- stream.close()
+ })
+ result.toList
+ } else {
+ val stream = Files.list(path)
+ try {
+ stream.iterator().asScala
+ .filter(p => Files.isRegularFile(p))
+ .filter { p =>
+ val base = p.getFileName.toString
+ specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext))
+ }
+ .toList
+ } finally {
+ stream.close()
+ }
}
} else {
List.empty