Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
Add replaces discovery mechanism for idempotent re-ingestion
riley022708b3ab947c27
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scalaindex 0dc26f47..30215ee9 100644--- a/memory-layer/src/main/scala/ix/memory/Main.scala+++ b/memory-layer/src/main/scala/ix/memory/Main.scala@@ -31,7 +31,7 @@ object Main extends IOApp.Simple {// 3. ServicesparserRouter = new ParserRouter()- ingestionService = new IngestionService(parserRouter, writeApi)+ ingestionService = new IngestionService(parserRouter, writeApi, queryApi)seeder = new GraphSeeder(queryApi)expander = new GraphExpander(queryApi)claimCollector = new ClaimCollector(queryApi)diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scalaindex d6ec84a8..a9e42b41 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala@@ -176,6 +176,18 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {)}+ override def getPatchesBySource(sourceUri: String, extractor: String): IO[Vector[Json]] =+ client.query(+ """FOR p IN patches+ | FILTER p.data.source.uri == @uri AND p.data.source.extractor == @extractor+ | SORT p.rev DESC+ | RETURN p""".stripMargin,+ Map(+ "uri" -> sourceUri.asInstanceOf[AnyRef],+ "extractor" -> extractor.asInstanceOf[AnyRef]+ )+ ).map(_.toVector)+// ── JSON Parsers (snake_case → camelCase) ───────────────────────────private def parseNode(json: Json): Option[GraphNode] = {diff --git a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scalaindex 66e46dd9..d75b735b 100644--- a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala@@ -14,6 +14,7 @@ trait GraphQueryApi {def getClaims(entityId: NodeId): IO[Vector[Claim]]def getLatestRev: IO[Rev]def getPatchesForEntity(entityId: NodeId): IO[List[Json]]+ def getPatchesBySource(sourceUri: String, extractor: String): IO[Vector[Json]]}sealed trait Directiondiff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 7f2e4cf3..d37f2854 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -4,14 +4,14 @@ import java.nio.file.{Files, Path}import cats.effect.IOimport cats.syntax.all._-import ix.memory.db.{CommitResult, GraphWriteApi}-import ix.memory.model.Rev+import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}+import ix.memory.model.{PatchId, Rev}/*** Orchestrates file ingestion: reads source files, parses them,* builds graph patches, and commits them through the write API.*/-class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {+class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {/*** Ingest a single file: read, parse, build patch, commit.@@ -32,7 +32,18 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {.map("%02x".format(_)).mkString)- patch = GraphPatchBuilder.build(filePath.toString, hash, result)+ // Check for existing patch from same source+extractor+ existing <- queryApi.getPatchesBySource(filePath.toString, "tree-sitter-python/1.0")+ prevPatch = existing.headOption+ prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)+ // If hash unchanged, skip (idempotent)+ _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))+ else IO.unit+ prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)+ .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)+ .map(PatchId(_))+ replaces = prevPatchId.toVector+ patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)_ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))commit <- writeApi.commitPatch(patch)} yield commit@@ -55,10 +66,15 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {files <- discoverFiles(path, language, recursive)results <- files.traverse(f => ingestFile(f).attempt)successes = results.collect { case Right(cr) => cr }+ idempotentCount = results.count {+ case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true+ case _ => false+ }latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))} yield IngestionResult(filesProcessed = files.size,patchesApplied = successes.size,+ filesSkipped = idempotentCount,entitiesCreated = 0, // Would need to count from parse resultslatestRev = latestRev)@@ -98,6 +114,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {case class IngestionResult(filesProcessed: Int,patchesApplied: Int,+ filesSkipped: Int,entitiesCreated: Int,latestRev: Rev)diff --git a/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala b/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scalaindex c0a4e5d4..c76a9966 100644--- a/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala+++ b/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala@@ -60,7 +60,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {val writeApi = new ArangoGraphWriteApi(client)val queryApi = new ArangoGraphQueryApi(client)val parserRouter = new ParserRouter()- val ingestion = new IngestionService(parserRouter, writeApi)+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)val contextService = buildContextService(queryApi)for {@@ -105,7 +105,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {val writeApi = new ArangoGraphWriteApi(client)val queryApi = new ArangoGraphQueryApi(client)val parserRouter = new ParserRouter()- val ingestion = new IngestionService(parserRouter, writeApi)+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)val contextService = buildContextService(queryApi)for {@@ -254,7 +254,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {val writeApi = new ArangoGraphWriteApi(client)val queryApi = new ArangoGraphQueryApi(client)val parserRouter = new ParserRouter()- val ingestion = new IngestionService(parserRouter, writeApi)+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)for {_ <- client.ensureSchema()@@ -267,8 +267,8 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {funcsBefore <- queryApi.findNodesByKind(NodeKind.Function)totalBefore = nodesBefore.size + funcsBefore.size- // Step 2: Re-ingest the same file with the same content- commit2 <- ingestion.ingestFile(fixtureFilePath)+ // Step 2: Re-ingest the same file with the same content — should be skipped (idempotent)+ result2 <- ingestion.ingestFile(fixtureFilePath).attempt// Count nodes after second ingestionnodesAfter <- queryApi.findNodesByKind(NodeKind.Class)@@ -278,10 +278,11 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {// First ingestion should succeedcommit1.status shouldBe CommitStatus.Ok- // Second ingestion should also succeed (new patchId, but upserts are used)- commit2.status shouldBe CommitStatus.Ok+ // Second ingestion should be skipped — hash unchanged+ result2.isLeft shouldBe true+ result2.left.exists(_.getMessage.startsWith("Idempotent")) shouldBe true- // Node count should NOT increase (UPSERT uses deterministic keys)+ // Node count should NOT increasetotalAfter shouldBe totalBefore// The nodes should have the same IDs (deterministic UUID generation)diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scalaindex 2bbcde78..3e4c695e 100644--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala@@ -62,7 +62,7 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {new ConfidenceScorerImpl(),new ConflictDetectorImpl())- val ingestionService = new IngestionService(new ParserRouter(), writeApi)+ val ingestionService = new IngestionService(new ParserRouter(), writeApi, queryApi)Routes.all(contextService, ingestionService, queryApi, conflictService, client).orNotFound}