Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
test: large repo ingestion integration tests + fix RoutesSpec compile
riley0227de093c71464e
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scalaindex 37803ee9..96c29b7a 100644--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala@@ -18,7 +18,7 @@ import ix.memory.TestDbHelperimport ix.memory.conflict.ConflictServiceimport ix.memory.context._import ix.memory.db._-import ix.memory.ingestion.{IngestionService, ParserRouter}+import ix.memory.ingestion.{BulkIngestionService, IngestionService, ParserRouter}import ix.memory.model._class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestDbHelper {@@ -64,7 +64,9 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestDnew ConflictDetectorImpl())val ingestionService = new IngestionService(new ParserRouter(), writeApi, queryApi)- Routes.all(contextService, ingestionService, queryApi, writeApi, conflictService, client).orNotFound+ val bulkWriteApi = new BulkWriteApi(client)+ val bulkIngestionService = new BulkIngestionService(new ParserRouter(), bulkWriteApi, queryApi)+ Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client).orNotFound}// ── Test 1: Health check ─────────────────────────────────────────────diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scalanew file mode 100644index 00000000..a115a40a--- /dev/null+++ b/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scala@@ -0,0 +1,104 @@+package ix.memory.ingestion++import java.nio.file.{Files, Path}++import cats.effect.IO+import cats.effect.testing.scalatest.AsyncIOSpec+import org.scalatest.flatspec.AsyncFlatSpec+import org.scalatest.matchers.should.Matchers++import ix.memory.TestDbHelper+import ix.memory.db.{ArangoClient, ArangoGraphQueryApi, BulkWriteApi}++class LargeRepoIngestionSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestDbHelper {++ val clientResource = ArangoClient.resource(+ host = "localhost", port = 8529,+ database = "ix_memory_test", user = "root", password = ""+ )++ private def createSyntheticRepo(fileCount: Int): IO[Path] = IO.blocking {+ val tmpDir = Files.createTempDirectory("ix-large-repo-test")+ for (i <- 0 until fileCount) {+ val subDir = tmpDir.resolve(s"pkg${i / 100}")+ Files.createDirectories(subDir)+ val content = s"class Service$i:\n def process(self):\n return $i\n"+ Files.writeString(subDir.resolve(s"service_$i.py"), content)+ }+ // Create one oversized file to test the size guard+ val bigFile = tmpDir.resolve("huge_generated.py")+ val bigContent = "x" * (1024 * 1024 + 1) // Just over 1MB+ Files.writeString(bigFile, bigContent)+ tmpDir+ }++ private def cleanupDir(dir: Path): IO[Unit] = IO.blocking {+ import scala.jdk.CollectionConverters._+ Files.walk(dir).iterator().asScala.toList.sortBy(_.toString).reverse.foreach(Files.deleteIfExists)+ }++ "BulkIngestionService" should "handle 500 files without OOM" in {+ clientResource.use { client =>+ val queryApi = new ArangoGraphQueryApi(client)+ val writeApi = new BulkWriteApi(client)+ val router = new ParserRouter()+ val service = new BulkIngestionService(router, writeApi, queryApi)++ for {+ _ <- client.ensureSchema()+ _ <- cleanDatabase(client)+ tmpDir <- createSyntheticRepo(500)+ result <- service.ingestPath(tmpDir, Some("python"), recursive = true)+ _ <- cleanupDir(tmpDir)+ } yield {+ result.filesProcessed should be >= 500+ result.patchesApplied should be >= 500+ result.skipReasons.tooLarge shouldBe 1 // the oversized file+ result.skipReasons.parseError shouldBe 0+ }+ }+ }++ it should "produce correct skip reasons on re-ingest" in {+ clientResource.use { client =>+ val queryApi = new ArangoGraphQueryApi(client)+ val writeApi = new BulkWriteApi(client)+ val router = new ParserRouter()+ val service = new BulkIngestionService(router, writeApi, queryApi)++ for {+ _ <- client.ensureSchema()+ _ <- cleanDatabase(client)+ tmpDir <- createSyntheticRepo(10)+ r1 <- service.ingestPath(tmpDir, Some("python"), recursive = true)+ r2 <- service.ingestPath(tmpDir, Some("python"), recursive = true)+ _ <- cleanupDir(tmpDir)+ } yield {+ r1.patchesApplied should be >= 10+ r2.patchesApplied shouldBe 0+ r2.skipReasons.unchanged should be >= 10+ }+ }+ }++ it should "survive chunked writes on medium batch" in {+ clientResource.use { client =>+ val queryApi = new ArangoGraphQueryApi(client)+ val writeApi = new BulkWriteApi(client)+ val router = new ParserRouter()+ val service = new BulkIngestionService(router, writeApi, queryApi)++ for {+ _ <- client.ensureSchema()+ _ <- cleanDatabase(client)+ tmpDir <- createSyntheticRepo(250)+ result <- service.ingestPath(tmpDir, Some("python"), recursive = true)+ rev <- queryApi.getLatestRev+ _ <- cleanupDir(tmpDir)+ } yield {+ result.patchesApplied should be >= 250+ rev.value should be > 0L+ }+ }+ }+}