Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: add batch source hash lookup for incremental ingestion skip

riley0227fe0837b17e2e
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
index c773c10f..2d75bc3c 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
@@ -308,6 +308,30 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {
scala.util.Try(java.util.UUID.fromString(s)).toOption.map(NodeId(_))
).toVector)
+ override def getSourceHashes(sourceUris: Seq[String]): IO[Map[String, String]] = {
+ if (sourceUris.isEmpty) IO.pure(Map.empty)
+ else {
+ val uriList = new java.util.ArrayList[String](sourceUris.size)
+ sourceUris.foreach(uriList.add)
+ client.query(
+ """FOR p IN patches
+ | FILTER p.data.source.uri IN @uris
+ | SORT p.rev DESC
+ | COLLECT uri = p.data.source.uri INTO groups
+ | LET latest = FIRST(groups)
+ | RETURN { uri: uri, hash: latest.p.data.source.sourceHash }""".stripMargin,
+ Map("uris" -> uriList.asInstanceOf[AnyRef])
+ ).map { results =>
+ results.flatMap { json =>
+ for {
+ uri <- json.hcursor.get[String]("uri").toOption
+ hash <- json.hcursor.get[String]("hash").toOption
+ } yield uri -> hash
+ }.toMap
+ }
+ }
+ }
+
// ── JSON Parsers (snake_case → camelCase) ───────────────────────────
private def parseNode(json: Json): Option[GraphNode] = {
diff --git a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
index 044c0417..e8c32c79 100644
--- a/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/GraphQueryApi.scala
@@ -20,6 +20,7 @@ trait GraphQueryApi {
def getPatchesBySource(sourceUri: String, extractor: String): IO[Vector[Json]]
def getChangedEntities(fromRev: Rev, toRev: Rev): IO[Vector[GraphNode]]
def resolvePrefix(prefix: String): IO[Vector[NodeId]]
+ def getSourceHashes(sourceUris: Seq[String]): IO[Map[String, String]]
}
sealed trait Direction
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index 977e7fd9..27a95d21 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -90,14 +90,10 @@ class BulkIngestionService(
} yield result
}
- /**
- * Load existing source hashes from the database for quick local comparison.
- * Returns Map[filePath -> hash].
- * TODO: Will be populated by Task 5 (getSourceHashes method on GraphQueryApi).
- * For now, returns empty map — every file is treated as changed.
- */
- private def loadExistingHashes(files: List[Path]): IO[Map[String, String]] =
- IO.pure(Map.empty[String, String])
+ private def loadExistingHashes(files: List[Path]): IO[Map[String, String]] = {
+ val paths = files.map(_.toString)
+ queryApi.getSourceHashes(paths)
+ }
private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
val json = Json.obj(