Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

Add replaces discovery mechanism for idempotent re-ingestion

riley022708b3ab947c27
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scala
index 0dc26f47..30215ee9 100644
--- a/memory-layer/src/main/scala/ix/memory/Main.scala
+++ b/memory-layer/src/main/scala/ix/memory/Main.scala
@@ -31,7 +31,7 @@ object Main extends IOApp.Simple {
// 3. Services
parserRouter = new ParserRouter()
- ingestionService = new IngestionService(parserRouter, writeApi)
+ ingestionService = new IngestionService(parserRouter, writeApi, queryApi)
seeder = new GraphSeeder(queryApi)
expander = new GraphExpander(queryApi)
claimCollector = new ClaimCollector(queryApi)
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
index d6ec84a8..a9e42b41 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
@@ -176,6 +176,18 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {
)
}
+ override def getPatchesBySource(sourceUri: String, extractor: String): IO[Vector[Json]] =
+ client.query(
+ """FOR p IN patches
+ | FILTER p.data.source.uri == @uri AND p.data.source.extractor == @extractor
+ | SORT p.rev DESC
+ | RETURN p""".stripMargin,
+ Map(
+ "uri" -> sourceUri.asInstanceOf[AnyRef],
+ "extractor" -> extractor.asInstanceOf[AnyRef]
+ )
+ ).map(_.toVector)
+
// ── JSON Parsers (snake_case → camelCase) ───────────────────────────
private def parseNode(json: Json): Option[GraphNode] = {
diff --git a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
index 66e46dd9..d75b735b 100644
--- a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
@@ -14,6 +14,7 @@ trait GraphQueryApi {
def getClaims(entityId: NodeId): IO[Vector[Claim]]
def getLatestRev: IO[Rev]
def getPatchesForEntity(entityId: NodeId): IO[List[Json]]
+ def getPatchesBySource(sourceUri: String, extractor: String): IO[Vector[Json]]
}
sealed trait Direction
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 7f2e4cf3..d37f2854 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -4,14 +4,14 @@ import java.nio.file.{Files, Path}
import cats.effect.IO
import cats.syntax.all._
-import ix.memory.db.{CommitResult, GraphWriteApi}
-import ix.memory.model.Rev
+import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}
+import ix.memory.model.{PatchId, Rev}
/**
* Orchestrates file ingestion: reads source files, parses them,
* builds graph patches, and commits them through the write API.
*/
-class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {
+class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {
/**
* Ingest a single file: read, parse, build patch, commit.
@@ -32,7 +32,18 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {
.map("%02x".format(_))
.mkString
)
- patch = GraphPatchBuilder.build(filePath.toString, hash, result)
+ // Check for existing patch from same source+extractor
+ existing <- queryApi.getPatchesBySource(filePath.toString, "tree-sitter-python/1.0")
+ prevPatch = existing.headOption
+ prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)
+ // If hash unchanged, skip (idempotent)
+ _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))
+ else IO.unit
+ prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)
+ .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)
+ .map(PatchId(_))
+ replaces = prevPatchId.toVector
+ patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)
_ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
commit <- writeApi.commitPatch(patch)
} yield commit
@@ -55,10 +66,15 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {
files <- discoverFiles(path, language, recursive)
results <- files.traverse(f => ingestFile(f).attempt)
successes = results.collect { case Right(cr) => cr }
+ idempotentCount = results.count {
+ case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true
+ case _ => false
+ }
latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))
} yield IngestionResult(
filesProcessed = files.size,
patchesApplied = successes.size,
+ filesSkipped = idempotentCount,
entitiesCreated = 0, // Would need to count from parse results
latestRev = latestRev
)
@@ -98,6 +114,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi) {
case class IngestionResult(
filesProcessed: Int,
patchesApplied: Int,
+ filesSkipped: Int,
entitiesCreated: Int,
latestRev: Rev
)
diff --git a/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala b/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala
index c0a4e5d4..c76a9966 100644
--- a/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/EndToEndSpec.scala
@@ -60,7 +60,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
val writeApi = new ArangoGraphWriteApi(client)
val queryApi = new ArangoGraphQueryApi(client)
val parserRouter = new ParserRouter()
- val ingestion = new IngestionService(parserRouter, writeApi)
+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)
val contextService = buildContextService(queryApi)
for {
@@ -105,7 +105,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
val writeApi = new ArangoGraphWriteApi(client)
val queryApi = new ArangoGraphQueryApi(client)
val parserRouter = new ParserRouter()
- val ingestion = new IngestionService(parserRouter, writeApi)
+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)
val contextService = buildContextService(queryApi)
for {
@@ -254,7 +254,7 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
val writeApi = new ArangoGraphWriteApi(client)
val queryApi = new ArangoGraphQueryApi(client)
val parserRouter = new ParserRouter()
- val ingestion = new IngestionService(parserRouter, writeApi)
+ val ingestion = new IngestionService(parserRouter, writeApi, queryApi)
for {
_ <- client.ensureSchema()
@@ -267,8 +267,8 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
funcsBefore <- queryApi.findNodesByKind(NodeKind.Function)
totalBefore = nodesBefore.size + funcsBefore.size
- // Step 2: Re-ingest the same file with the same content
- commit2 <- ingestion.ingestFile(fixtureFilePath)
+ // Step 2: Re-ingest the same file with the same content — should be skipped (idempotent)
+ result2 <- ingestion.ingestFile(fixtureFilePath).attempt
// Count nodes after second ingestion
nodesAfter <- queryApi.findNodesByKind(NodeKind.Class)
@@ -278,10 +278,11 @@ class EndToEndSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
// First ingestion should succeed
commit1.status shouldBe CommitStatus.Ok
- // Second ingestion should also succeed (new patchId, but upserts are used)
- commit2.status shouldBe CommitStatus.Ok
+ // Second ingestion should be skipped — hash unchanged
+ result2.isLeft shouldBe true
+ result2.left.exists(_.getMessage.startsWith("Idempotent")) shouldBe true
- // Node count should NOT increase (UPSERT uses deterministic keys)
+ // Node count should NOT increase
totalAfter shouldBe totalBefore
// The nodes should have the same IDs (deterministic UUID generation)
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
index 2bbcde78..3e4c695e 100644
--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
@@ -62,7 +62,7 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
new ConfidenceScorerImpl(),
new ConflictDetectorImpl()
)
- val ingestionService = new IngestionService(new ParserRouter(), writeApi)
+ val ingestionService = new IngestionService(new ParserRouter(), writeApi, queryApi)
Routes.all(contextService, ingestionService, queryApi, conflictService, client).orNotFound
}