Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix: skip node_modules, .git, target and other ignored dirs during file discovery
riley02277f56b65eeb69
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalaindex 27a95d21..41e520eb 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -135,16 +135,41 @@ class BulkIngestionService(if (specialFiles.contains(base) || extensions.exists(base.endsWith)) List(path)else List.empty} else if (Files.isDirectory(path)) {- val stream = if (recursive) Files.walk(path) else Files.list(path)- try {- stream.iterator().asScala- .filter(Files.isRegularFile(_))- .filter { p =>- val base = p.getFileName.toString- specialFiles.contains(base) || extensions.exists(base.endsWith)+ if (recursive) {+ val ignoredDirs = Set(+ "node_modules", ".git", "target", "dist", "build", ".next",+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",+ ".gradle", ".idea", ".vscode", ".settings", "out",+ ".cache", ".parcel-cache", "coverage", ".nyc_output"+ )+ val result = scala.collection.mutable.ListBuffer.empty[Path]+ Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {+ override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ if (dir != path && ignoredDirs.contains(dir.getFileName.toString))+ java.nio.file.FileVisitResult.SKIP_SUBTREE+ else+ java.nio.file.FileVisitResult.CONTINUE}- .toList- } finally stream.close()+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ val base = file.getFileName.toString+ if (specialFiles.contains(base) || extensions.exists(base.endsWith))+ result += file+ java.nio.file.FileVisitResult.CONTINUE+ }+ })+ result.toList+ } else {+ val stream = Files.list(path)+ try {+ stream.iterator().asScala+ .filter(Files.isRegularFile(_))+ .filter { p =>+ val base = p.getFileName.toString+ specialFiles.contains(base) || extensions.exists(base.endsWith)+ }+ .toList+ } finally stream.close()+ }} else List.empty}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 7390d36d..920b6854 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -133,26 +133,43 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, querif (isSpecial || extensions.exists(ext => base.endsWith(ext))) List(path)else List.empty} else if (Files.isDirectory(path)) {- val stream = if (recursive) Files.walk(path) else Files.list(path)- try {- stream.iterator().asScala- .filter(p => Files.isRegularFile(p))- .filter { p =>- val base = p.getFileName.toString- val isSpecial = Set(- "Dockerfile",- "Makefile",- "README",- "LICENSE",- "NOTICE",- "build.sbt"- ).contains(base)- val matchesExt = extensions.exists(ext => base.endsWith(ext))- matchesExt || isSpecial+ val specialFiles = Set("Dockerfile", "Makefile", "README", "LICENSE", "NOTICE", "build.sbt")+ if (recursive) {+ val ignoredDirs = Set(+ "node_modules", ".git", "target", "dist", "build", ".next",+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache",+ ".gradle", ".idea", ".vscode", ".settings", "out",+ ".cache", ".parcel-cache", "coverage", ".nyc_output"+ )+ val result = scala.collection.mutable.ListBuffer.empty[Path]+ Files.walkFileTree(path, new java.nio.file.SimpleFileVisitor[Path] {+ override def preVisitDirectory(dir: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ if (dir != path && ignoredDirs.contains(dir.getFileName.toString))+ java.nio.file.FileVisitResult.SKIP_SUBTREE+ else+ java.nio.file.FileVisitResult.CONTINUE+ }+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ val base = file.getFileName.toString+ if (specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext)))+ result += file+ java.nio.file.FileVisitResult.CONTINUE}- .toList- } finally {- stream.close()+ })+ result.toList+ } else {+ val stream = Files.list(path)+ try {+ stream.iterator().asScala+ .filter(p => Files.isRegularFile(p))+ .filter { p =>+ val base = p.getFileName.toString+ specialFiles.contains(base) || extensions.exists(ext => base.endsWith(ext))+ }+ .toList+ } finally {+ stream.close()+ }}} else {List.empty