Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: add GraphWriteApi with patch commit, MVCC, and idempotency

riley0227521093db3c2b
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala
new file mode 100644
index 00000000..0e2fff9f
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala
@@ -0,0 +1,317 @@
+package ix.memory.db
+
+import java.time.Instant
+import java.util.UUID
+
+import cats.effect.IO
+import cats.syntax.traverse._
+import cats.syntax.foldable._
+import cats.instances.vector._
+import io.circe.syntax._
+import io.circe.Json
+
+import ix.memory.model._
+
+class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {
+
+ override def commitPatch(patch: GraphPatch): IO[CommitResult] =
+ for {
+ // Step 1: Check idempotency — has this patch_id already been applied?
+ idempotent <- checkIdempotency(patch.patchId, patch.tenant)
+ result <- idempotent match {
+ case Some(storedRev) =>
+ IO.pure(CommitResult(storedRev, CommitStatus.Idempotent))
+ case None =>
+ doCommit(patch)
+ }
+ } yield result
+
+ private def checkIdempotency(patchId: PatchId, tenant: TenantId): IO[Option[Rev]] =
+ client.queryOne(
+ """FOR ik IN idempotency_keys
+ | FILTER ik.key == @key
+ | RETURN ik.rev""".stripMargin,
+ Map("key" -> idempotencyKey(patchId, tenant).asInstanceOf[AnyRef])
+ ).map(_.flatMap(_.as[Long].toOption).map(Rev(_)))
+
+ private def idempotencyKey(patchId: PatchId, tenant: TenantId): String =
+ s"${tenant.value}:${patchId.value}"
+
+ private def doCommit(patch: GraphPatch): IO[CommitResult] =
+ for {
+ // Step 2: Load latest_rev for tenant
+ latestRev <- loadLatestRev(patch.tenant)
+
+ // Step 3: Check baseRev (if > 0, must match latest)
+ result <- {
+ if (patch.baseRev.value > 0L && patch.baseRev.value != latestRev) {
+ IO.pure(CommitResult(Rev(latestRev), CommitStatus.BaseRevMismatch))
+ } else {
+ val newRev = latestRev + 1L
+ for {
+ // Step 5: Execute each op
+ _ <- patch.ops.traverse_(op => executeOp(op, patch, newRev))
+ // Step 6: Store patch
+ _ <- storePatch(patch, newRev)
+ // Step 7: Store idempotency key
+ _ <- storeIdempotencyKey(patch.patchId, patch.tenant, newRev)
+ // Step 8: Update tenant revision
+ _ <- updateTenantRevision(patch.tenant, newRev)
+ } yield CommitResult(Rev(newRev), CommitStatus.Ok)
+ }
+ }
+ } yield result
+
+ private def loadLatestRev(tenant: TenantId): IO[Long] =
+ client.queryOne(
+ """FOR r IN revisions
+ | FILTER r._key == @key
+ | RETURN r.rev""".stripMargin,
+ Map("key" -> tenant.value.toString.asInstanceOf[AnyRef])
+ ).map(_.flatMap(_.as[Long].toOption).getOrElse(0L))
+
+ private def executeOp(op: PatchOp, patch: GraphPatch, newRev: Long): IO[Unit] = op match {
+ case PatchOp.UpsertNode(id, kind, name, attrs) =>
+ val now = Instant.now().toString
+ val provenanceJson = buildProvenance(patch)
+ val attrsJson = Json.obj(attrs.toSeq.map { case (k, v) => k -> v }: _*).noSpaces
+ client.execute(
+ """UPSERT { _key: @key }
+ | INSERT {
+ | _key: @key,
+ | id: @id,
+ | kind: @kind,
+ | name: @name,
+ | tenant: @tenant,
+ | attrs: @attrs,
+ | created_rev: @created_rev,
+ | deleted_rev: null,
+ | provenance: @provenance,
+ | created_at: @now,
+ | updated_at: @now
+ | }
+ | UPDATE {
+ | kind: @kind,
+ | name: @name,
+ | attrs: @attrs,
+ | deleted_rev: null,
+ | provenance: @provenance,
+ | updated_at: @now
+ | }
+ | IN nodes""".stripMargin,
+ Map(
+ "key" -> id.value.toString.asInstanceOf[AnyRef],
+ "id" -> id.value.toString.asInstanceOf[AnyRef],
+ "kind" -> nodeKindToString(kind).asInstanceOf[AnyRef],
+ "name" -> name.asInstanceOf[AnyRef],
+ "tenant" -> patch.tenant.value.toString.asInstanceOf[AnyRef],
+ "attrs" -> parseToJavaMap(attrsJson).asInstanceOf[AnyRef],
+ "created_rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "provenance" -> parseToJavaMap(provenanceJson.noSpaces).asInstanceOf[AnyRef],
+ "now" -> now.asInstanceOf[AnyRef]
+ )
+ )
+
+ case PatchOp.UpsertEdge(id, src, dst, predicate, attrs) =>
+ val now = Instant.now().toString
+ val provenanceJson = buildProvenance(patch)
+ val attrsJson = Json.obj(attrs.toSeq.map { case (k, v) => k -> v }: _*).noSpaces
+ client.execute(
+ """UPSERT { _key: @key }
+ | INSERT {
+ | _key: @key,
+ | _from: @from,
+ | _to: @to,
+ | id: @id,
+ | src: @src,
+ | dst: @dst,
+ | predicate: @predicate,
+ | tenant: @tenant,
+ | attrs: @attrs,
+ | created_rev: @created_rev,
+ | deleted_rev: null,
+ | provenance: @provenance,
+ | created_at: @now,
+ | updated_at: @now
+ | }
+ | UPDATE {
+ | _from: @from,
+ | _to: @to,
+ | src: @src,
+ | dst: @dst,
+ | predicate: @predicate,
+ | attrs: @attrs,
+ | deleted_rev: null,
+ | provenance: @provenance,
+ | updated_at: @now
+ | }
+ | IN edges""".stripMargin,
+ Map(
+ "key" -> id.value.toString.asInstanceOf[AnyRef],
+ "id" -> id.value.toString.asInstanceOf[AnyRef],
+ "from" -> s"nodes/${src.value}".asInstanceOf[AnyRef],
+ "to" -> s"nodes/${dst.value}".asInstanceOf[AnyRef],
+ "src" -> src.value.toString.asInstanceOf[AnyRef],
+ "dst" -> dst.value.toString.asInstanceOf[AnyRef],
+ "predicate" -> predicate.value.asInstanceOf[AnyRef],
+ "tenant" -> patch.tenant.value.toString.asInstanceOf[AnyRef],
+ "attrs" -> parseToJavaMap(attrsJson).asInstanceOf[AnyRef],
+ "created_rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "provenance" -> parseToJavaMap(provenanceJson.noSpaces).asInstanceOf[AnyRef],
+ "now" -> now.asInstanceOf[AnyRef]
+ )
+ )
+
+ case PatchOp.DeleteNode(id) =>
+ client.execute(
+ """FOR n IN nodes
+ | FILTER n._key == @key
+ | UPDATE n WITH { deleted_rev: @deleted_rev } IN nodes""".stripMargin,
+ Map(
+ "key" -> id.value.toString.asInstanceOf[AnyRef],
+ "deleted_rev" -> Long.box(newRev).asInstanceOf[AnyRef]
+ )
+ )
+
+ case PatchOp.DeleteEdge(id) =>
+ client.execute(
+ """FOR e IN edges
+ | FILTER e._key == @key
+ | UPDATE e WITH { deleted_rev: @deleted_rev } IN edges""".stripMargin,
+ Map(
+ "key" -> id.value.toString.asInstanceOf[AnyRef],
+ "deleted_rev" -> Long.box(newRev).asInstanceOf[AnyRef]
+ )
+ )
+
+ case PatchOp.AssertClaim(entityId, field, value, confidence) =>
+ val claimKey = UUID.randomUUID().toString
+ val provenanceJson = buildProvenance(patch)
+ client.execute(
+ """INSERT {
+ | _key: @key,
+ | entity_id: @entity_id,
+ | field: @field,
+ | value: @value,
+ | confidence: @confidence,
+ | status: @status,
+ | tenant: @tenant,
+ | created_rev: @created_rev,
+ | deleted_rev: null,
+ | provenance: @provenance
+ |} INTO claims""".stripMargin,
+ Map(
+ "key" -> claimKey.asInstanceOf[AnyRef],
+ "entity_id" -> entityId.value.toString.asInstanceOf[AnyRef],
+ "field" -> field.asInstanceOf[AnyRef],
+ "value" -> jsonToJava(value).asInstanceOf[AnyRef],
+ "confidence" -> confidence.map(Double.box).orNull.asInstanceOf[AnyRef],
+ "status" -> "active".asInstanceOf[AnyRef],
+ "tenant" -> patch.tenant.value.toString.asInstanceOf[AnyRef],
+ "created_rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "provenance" -> parseToJavaMap(provenanceJson.noSpaces).asInstanceOf[AnyRef]
+ )
+ )
+
+ case PatchOp.RetractClaim(claimId) =>
+ client.execute(
+ """FOR c IN claims
+ | FILTER c._key == @key
+ | UPDATE c WITH { status: "retracted", deleted_rev: @deleted_rev } IN claims""".stripMargin,
+ Map(
+ "key" -> claimId.value.toString.asInstanceOf[AnyRef],
+ "deleted_rev" -> Long.box(newRev).asInstanceOf[AnyRef]
+ )
+ )
+ }
+
+ private def storePatch(patch: GraphPatch, newRev: Long): IO[Unit] = {
+ val patchJson = patch.asJson.noSpaces
+ client.execute(
+ """INSERT {
+ | _key: @key,
+ | patch_id: @patch_id,
+ | tenant: @tenant,
+ | rev: @rev,
+ | data: @data
+ |} INTO patches""".stripMargin,
+ Map(
+ "key" -> s"${patch.tenant.value}:${patch.patchId.value}".asInstanceOf[AnyRef],
+ "patch_id" -> patch.patchId.value.toString.asInstanceOf[AnyRef],
+ "tenant" -> patch.tenant.value.toString.asInstanceOf[AnyRef],
+ "rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "data" -> parseToJavaMap(patchJson).asInstanceOf[AnyRef]
+ )
+ )
+ }
+
+ private def storeIdempotencyKey(patchId: PatchId, tenant: TenantId, newRev: Long): IO[Unit] =
+ client.execute(
+ """INSERT {
+ | key: @key,
+ | rev: @rev,
+ | created_at: DATE_NOW() / 1000
+ |} INTO idempotency_keys""".stripMargin,
+ Map(
+ "key" -> idempotencyKey(patchId, tenant).asInstanceOf[AnyRef],
+ "rev" -> Long.box(newRev).asInstanceOf[AnyRef]
+ )
+ )
+
+ private def updateTenantRevision(tenant: TenantId, newRev: Long): IO[Unit] =
+ client.execute(
+ """UPSERT { _key: @key }
+ | INSERT { _key: @key, rev: @rev }
+ | UPDATE { rev: @rev }
+ | IN revisions""".stripMargin,
+ Map(
+ "key" -> tenant.value.toString.asInstanceOf[AnyRef],
+ "rev" -> Long.box(newRev).asInstanceOf[AnyRef]
+ )
+ )
+
+ private def buildProvenance(patch: GraphPatch): Json =
+ Json.obj(
+ "source_uri" -> Json.fromString(patch.source.uri),
+ "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
+ "extractor" -> Json.fromString(patch.source.extractor),
+ "source_type" -> Json.fromString(sourceTypeToString(patch.source.sourceType)),
+ "observed_at" -> Json.fromString(patch.timestamp.toString)
+ )
+
+ private def sourceTypeToString(st: SourceType): String = st match {
+ case SourceType.Code => "code"
+ case SourceType.Config => "config"
+ case SourceType.Doc => "doc"
+ case SourceType.Test => "test"
+ case SourceType.Schema => "schema"
+ case SourceType.Commit => "commit"
+ case SourceType.Comment => "comment"
+ case SourceType.Inferred => "inferred"
+ case SourceType.Human => "human"
+ }
+
+ private def nodeKindToString(nk: NodeKind): String = nk match {
+ case NodeKind.Module => "module"
+ case NodeKind.File => "file"
+ case NodeKind.Class => "class"
+ case NodeKind.Function => "function"
+ case NodeKind.Variable => "variable"
+ case NodeKind.Config => "config"
+ case NodeKind.ConfigEntry => "config_entry"
+ case NodeKind.Service => "service"
+ case NodeKind.Endpoint => "endpoint"
+ }
+
+ /** Convert a JSON string into a java.util.Map for use as an ArangoDB bind variable. */
+ private def parseToJavaMap(jsonStr: String): java.util.Map[String, AnyRef] = {
+ val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
+ mapper.readValue(jsonStr, classOf[java.util.Map[String, AnyRef]])
+ }
+
+ /** Convert a circe Json value to a Java object suitable for ArangoDB bind variables. */
+ private def jsonToJava(json: Json): AnyRef = {
+ val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
+ mapper.readValue(json.noSpaces, classOf[AnyRef])
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/db/GraphWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/GraphWriteApi.scala
new file mode 100644
index 00000000..2a79e36e
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/db/GraphWriteApi.scala
@@ -0,0 +1,17 @@
+package ix.memory.db
+
+import cats.effect.IO
+import ix.memory.model._
+
+trait GraphWriteApi {
+ def commitPatch(patch: GraphPatch): IO[CommitResult]
+}
+
+final case class CommitResult(newRev: Rev, status: CommitStatus)
+
+sealed trait CommitStatus
+object CommitStatus {
+ case object Ok extends CommitStatus
+ case object Idempotent extends CommitStatus
+ case object BaseRevMismatch extends CommitStatus
+}
diff --git a/memory-layer/src/test/scala/ix/memory/db/GraphWriteApiSpec.scala b/memory-layer/src/test/scala/ix/memory/db/GraphWriteApiSpec.scala
new file mode 100644
index 00000000..766e01a1
--- /dev/null
+++ b/memory-layer/src/test/scala/ix/memory/db/GraphWriteApiSpec.scala
@@ -0,0 +1,231 @@
+package ix.memory.db
+
+import java.time.Instant
+import java.util.UUID
+
+import cats.effect.IO
+import cats.effect.testing.scalatest.AsyncIOSpec
+import io.circe.Json
+import org.scalatest.flatspec.AsyncFlatSpec
+import org.scalatest.matchers.should.Matchers
+
+import ix.memory.model._
+
+class GraphWriteApiSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers {
+
+ val clientResource = ArangoClient.resource(
+ host = "localhost", port = 8529,
+ database = "ix_memory_test", user = "root", password = ""
+ )
+
+ private def makePatch(
+ tenant: TenantId,
+ baseRev: Rev = Rev(0L),
+ ops: Vector[PatchOp] = Vector.empty,
+ patchId: PatchId = PatchId(UUID.randomUUID())
+ ): GraphPatch =
+ GraphPatch(
+ patchId = patchId,
+ tenant = tenant,
+ actor = "test-actor",
+ timestamp = Instant.parse("2025-06-01T12:00:00Z"),
+ source = PatchSource(
+ uri = "test://source",
+ sourceHash = Some("hash123"),
+ extractor = "test-extractor",
+ sourceType = SourceType.Code
+ ),
+ baseRev = baseRev,
+ ops = ops,
+ replaces = Vector.empty,
+ intent = Some("test intent")
+ )
+
+ // ── Test 1: commit a patch and increment revision ──────────────────
+
+ "GraphWriteApi" should "commit a patch and increment revision" in {
+ clientResource.use { client =>
+ for {
+ _ <- client.ensureSchema()
+ api = new ArangoGraphWriteApi(client)
+ tenant = TenantId(UUID.randomUUID())
+ nodeId = NodeId(UUID.randomUUID())
+ patch = makePatch(
+ tenant = tenant,
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId,
+ kind = NodeKind.Function,
+ name = "testFunc",
+ attrs = Map("lang" -> Json.fromString("scala"))
+ )
+ )
+ )
+ result <- api.commitPatch(patch)
+ } yield {
+ result.status shouldBe CommitStatus.Ok
+ result.newRev.value should be > 0L
+ }
+ }
+ }
+
+ // ── Test 2: be idempotent on duplicate patch_id ────────────────────
+
+ it should "be idempotent on duplicate patch_id" in {
+ clientResource.use { client =>
+ for {
+ _ <- client.ensureSchema()
+ api = new ArangoGraphWriteApi(client)
+ tenant = TenantId(UUID.randomUUID())
+ nodeId = NodeId(UUID.randomUUID())
+ patchId = PatchId(UUID.randomUUID())
+ patch = makePatch(
+ tenant = tenant,
+ patchId = patchId,
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId,
+ kind = NodeKind.Module,
+ name = "testModule",
+ attrs = Map.empty[String, Json]
+ )
+ )
+ )
+ result1 <- api.commitPatch(patch)
+ result2 <- api.commitPatch(patch)
+ } yield {
+ result1.status shouldBe CommitStatus.Ok
+ result2.status shouldBe CommitStatus.Idempotent
+ result2.newRev shouldBe result1.newRev
+ }
+ }
+ }
+
+ // ── Test 3: reject on base_rev mismatch ────────────────────────────
+
+ it should "reject on base_rev mismatch" in {
+ clientResource.use { client =>
+ for {
+ _ <- client.ensureSchema()
+ api = new ArangoGraphWriteApi(client)
+ tenant = TenantId(UUID.randomUUID())
+ nodeId1 = NodeId(UUID.randomUUID())
+ // First patch to establish rev 1
+ patch1 = makePatch(
+ tenant = tenant,
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId1,
+ kind = NodeKind.Service,
+ name = "svc1",
+ attrs = Map.empty[String, Json]
+ )
+ )
+ )
+ _ <- api.commitPatch(patch1)
+ // Second patch with wrong baseRev
+ nodeId2 = NodeId(UUID.randomUUID())
+ patch2 = makePatch(
+ tenant = tenant,
+ baseRev = Rev(999L),
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId2,
+ kind = NodeKind.Service,
+ name = "svc2",
+ attrs = Map.empty[String, Json]
+ )
+ )
+ )
+ result <- api.commitPatch(patch2)
+ } yield {
+ result.status shouldBe CommitStatus.BaseRevMismatch
+ }
+ }
+ }
+
+ // ── Test 4: persist node visible via query ─────────────────────────
+
+ it should "persist node visible via query" in {
+ clientResource.use { client =>
+ for {
+ _ <- client.ensureSchema()
+ api = new ArangoGraphWriteApi(client)
+ tenant = TenantId(UUID.randomUUID())
+ nodeId = NodeId(UUID.randomUUID())
+ patch = makePatch(
+ tenant = tenant,
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId,
+ kind = NodeKind.File,
+ name = "Main.scala",
+ attrs = Map("path" -> Json.fromString("/src/Main.scala"))
+ )
+ )
+ )
+ _ <- api.commitPatch(patch)
+ result <- client.queryOne(
+ "FOR n IN nodes FILTER n._key == @key RETURN n",
+ Map("key" -> nodeId.value.toString.asInstanceOf[AnyRef])
+ )
+ } yield {
+ result shouldBe defined
+ val doc = result.get
+ doc.hcursor.get[String]("kind") shouldBe Right("file")
+ doc.hcursor.get[String]("name") shouldBe Right("Main.scala")
+ doc.hcursor.get[String]("tenant") shouldBe Right(tenant.value.toString)
+ // attrs should contain the path attribute
+ doc.hcursor.downField("attrs").downField("path").as[String] shouldBe Right("/src/Main.scala")
+ // deleted_rev should be null (not soft-deleted)
+ doc.hcursor.get[Option[Long]]("deleted_rev") shouldBe Right(None)
+ }
+ }
+ }
+
+ // ── Test 5: soft delete node via MVCC ──────────────────────────────
+
+ it should "soft delete node via MVCC" in {
+ clientResource.use { client =>
+ for {
+ _ <- client.ensureSchema()
+ api = new ArangoGraphWriteApi(client)
+ tenant = TenantId(UUID.randomUUID())
+ nodeId = NodeId(UUID.randomUUID())
+ // First patch: create the node
+ patch1 = makePatch(
+ tenant = tenant,
+ ops = Vector(
+ PatchOp.UpsertNode(
+ id = nodeId,
+ kind = NodeKind.Class,
+ name = "MyClass",
+ attrs = Map.empty[String, Json]
+ )
+ )
+ )
+ res1 <- api.commitPatch(patch1)
+ // Second patch: soft delete the node
+ patch2 = makePatch(
+ tenant = tenant,
+ baseRev = res1.newRev,
+ ops = Vector(PatchOp.DeleteNode(nodeId))
+ )
+ res2 <- api.commitPatch(patch2)
+ // Query the node to verify soft delete
+ result <- client.queryOne(
+ "FOR n IN nodes FILTER n._key == @key RETURN n",
+ Map("key" -> nodeId.value.toString.asInstanceOf[AnyRef])
+ )
+ } yield {
+ res1.status shouldBe CommitStatus.Ok
+ res2.status shouldBe CommitStatus.Ok
+ res2.newRev.value shouldBe res1.newRev.value + 1
+
+ result shouldBe defined
+ val doc = result.get
+ doc.hcursor.get[Long]("deleted_rev") shouldBe Right(res2.newRev.value)
+ }
+ }
+ }
+}