Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

test: large repo ingestion integration tests + fix RoutesSpec compile

riley0227de093c71464e
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
index 37803ee9..96c29b7a 100644
--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
@@ -18,7 +18,7 @@ import ix.memory.TestDbHelper
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
-import ix.memory.ingestion.{IngestionService, ParserRouter}
+import ix.memory.ingestion.{BulkIngestionService, IngestionService, ParserRouter}
import ix.memory.model._
class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestDbHelper {
@@ -64,7 +64,9 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD
new ConflictDetectorImpl()
)
val ingestionService = new IngestionService(new ParserRouter(), writeApi, queryApi)
- Routes.all(contextService, ingestionService, queryApi, writeApi, conflictService, client).orNotFound
+ val bulkWriteApi = new BulkWriteApi(client)
+ val bulkIngestionService = new BulkIngestionService(new ParserRouter(), bulkWriteApi, queryApi)
+ Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client).orNotFound
}
// ── Test 1: Health check ─────────────────────────────────────────────
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scala
new file mode 100644
index 00000000..a115a40a
--- /dev/null
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/LargeRepoIngestionSpec.scala
@@ -0,0 +1,104 @@
+package ix.memory.ingestion
+
+import java.nio.file.{Files, Path}
+
+import cats.effect.IO
+import cats.effect.testing.scalatest.AsyncIOSpec
+import org.scalatest.flatspec.AsyncFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import ix.memory.TestDbHelper
+import ix.memory.db.{ArangoClient, ArangoGraphQueryApi, BulkWriteApi}
+
+class LargeRepoIngestionSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestDbHelper {
+
+ val clientResource = ArangoClient.resource(
+ host = "localhost", port = 8529,
+ database = "ix_memory_test", user = "root", password = ""
+ )
+
+ private def createSyntheticRepo(fileCount: Int): IO[Path] = IO.blocking {
+ val tmpDir = Files.createTempDirectory("ix-large-repo-test")
+ for (i <- 0 until fileCount) {
+ val subDir = tmpDir.resolve(s"pkg${i / 100}")
+ Files.createDirectories(subDir)
+ val content = s"class Service$i:\n def process(self):\n return $i\n"
+ Files.writeString(subDir.resolve(s"service_$i.py"), content)
+ }
+ // Create one oversized file to test the size guard
+ val bigFile = tmpDir.resolve("huge_generated.py")
+ val bigContent = "x" * (1024 * 1024 + 1) // Just over 1MB
+ Files.writeString(bigFile, bigContent)
+ tmpDir
+ }
+
+ private def cleanupDir(dir: Path): IO[Unit] = IO.blocking {
+ import scala.jdk.CollectionConverters._
+ Files.walk(dir).iterator().asScala.toList.sortBy(_.toString).reverse.foreach(Files.deleteIfExists)
+ }
+
+ "BulkIngestionService" should "handle 500 files without OOM" in {
+ clientResource.use { client =>
+ val queryApi = new ArangoGraphQueryApi(client)
+ val writeApi = new BulkWriteApi(client)
+ val router = new ParserRouter()
+ val service = new BulkIngestionService(router, writeApi, queryApi)
+
+ for {
+ _ <- client.ensureSchema()
+ _ <- cleanDatabase(client)
+ tmpDir <- createSyntheticRepo(500)
+ result <- service.ingestPath(tmpDir, Some("python"), recursive = true)
+ _ <- cleanupDir(tmpDir)
+ } yield {
+ result.filesProcessed should be >= 500
+ result.patchesApplied should be >= 500
+ result.skipReasons.tooLarge shouldBe 1 // the oversized file
+ result.skipReasons.parseError shouldBe 0
+ }
+ }
+ }
+
+ it should "produce correct skip reasons on re-ingest" in {
+ clientResource.use { client =>
+ val queryApi = new ArangoGraphQueryApi(client)
+ val writeApi = new BulkWriteApi(client)
+ val router = new ParserRouter()
+ val service = new BulkIngestionService(router, writeApi, queryApi)
+
+ for {
+ _ <- client.ensureSchema()
+ _ <- cleanDatabase(client)
+ tmpDir <- createSyntheticRepo(10)
+ r1 <- service.ingestPath(tmpDir, Some("python"), recursive = true)
+ r2 <- service.ingestPath(tmpDir, Some("python"), recursive = true)
+ _ <- cleanupDir(tmpDir)
+ } yield {
+ r1.patchesApplied should be >= 10
+ r2.patchesApplied shouldBe 0
+ r2.skipReasons.unchanged should be >= 10
+ }
+ }
+ }
+
+ it should "survive chunked writes on medium batch" in {
+ clientResource.use { client =>
+ val queryApi = new ArangoGraphQueryApi(client)
+ val writeApi = new BulkWriteApi(client)
+ val router = new ParserRouter()
+ val service = new BulkIngestionService(router, writeApi, queryApi)
+
+ for {
+ _ <- client.ensureSchema()
+ _ <- cleanDatabase(client)
+ tmpDir <- createSyntheticRepo(250)
+ result <- service.ingestPath(tmpDir, Some("python"), recursive = true)
+ rev <- queryApi.getLatestRev
+ _ <- cleanupDir(tmpDir)
+ } yield {
+ result.patchesApplied should be >= 250
+ rev.value should be > 0L
+ }
+ }
+ }
+}