Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
perf(db): parallel chunk processing and compute-pool doc building
Tanner Kocher67168f0eded2
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 84f3d53a..b06736a7 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -62,6 +62,11 @@ class BulkWriteApi(client: ArangoClient) {}.map(_.sum)}+ /**+ * Build documents for a single chunk and insert them into ArangoDB.+ * Document building runs on the compute pool (IO.delay) in parallel+ * across FileBatches, then inserts run concurrently across collections.+ */def commitBatch(fileBatches: Vector[FileBatch],baseRev: Long,@@ -70,20 +75,25 @@ class BulkWriteApi(client: ArangoClient) {val newRev = baseRev + 1Lfor {- // Build all document maps in a single IO.blocking pass — one traversal instead of four.+ // Build documents in parallel across FileBatches on the compute pool.+ // Each FileBatch produces its own (nodes, edges, claims, patch) tuple independently.docsAndBuildMs <- timed {- IO.blocking {- val nodesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]- val edgesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]- val claimsBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]- val patchesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]- fileBatches.foreach { fb =>- nodesBuf ++= fb.nodeDocuments(newRev)- edgesBuf ++= fb.edgeDocuments(newRev)- claimsBuf ++= fb.claimDocuments(newRev)- patchesBuf += fb.patchDocument(newRev)- }- (nodesBuf.result(), edgesBuf.result(), claimsBuf.result(), patchesBuf.result())+ fileBatches.parTraverse { fb =>+ IO.delay {+ (fb.nodeDocuments(newRev), fb.edgeDocuments(newRev), fb.claimDocuments(newRev), fb.patchDocument(newRev))+ }+ }.map { results =>+ val nodesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]+ val edgesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]+ val claimsBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]+ val patchesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]+ results.foreach { case (nodes, edges, claims, patch) =>+ nodesBuf ++= nodes+ edgesBuf ++= edges+ claimsBuf ++= claims+ patchesBuf += patch+ }+ (nodesBuf.result(), edgesBuf.result(), claimsBuf.result(), patchesBuf.result())}}(docs, buildDocsMs) = docsAndBuildMs@@ -105,8 +115,7 @@ class BulkWriteApi(client: ArangoClient) {.both(insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs))).map { case (((n, e), c), p) => (n, e, c, p) }(insertNodesMs, insertEdgesMs, insertClaimsMs, insertPatchesMs) = allInsertMs- updateRevMs <- timedUnit(updateRevision(newRev))- totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs + updateRevMs+ totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs_ <- logChunkTiming(s"rev=$newRev $chunkSummary",totalMs,@@ -118,7 +127,6 @@ class BulkWriteApi(client: ArangoClient) {"insertEdges" -> insertEdgesMs,"insertClaims" -> insertClaimsMs,"insertPatches" -> insertPatchesMs,- "updateRevision" -> updateRevMs,))} yield CommitResult(Rev(newRev), CommitStatus.Ok)@@ -169,12 +177,15 @@ class BulkWriteApi(client: ArangoClient) {)/**- * Commit file batches in chunks. Each chunk gets its own revision.- * If a chunk fails, prior chunks are preserved (partial progress).+ * Commit file batches in chunks with pre-assigned revisions.+ *+ * Each chunk gets a unique revision assigned upfront (baseRev+1, baseRev+2, ...),+ * allowing all chunks to build documents and insert concurrently rather than+ * waiting for the previous chunk's revision to be committed. The revision counter+ * is updated once at the end to the final value.** When multiple chunks exist and baseRev > 0 (incremental update), tombstone+retire* runs once upfront for all files in parallel rather than once per chunk sequentially.- * This reduces 2*N AQL UPDATE scans to 2 — e.g. 160 → 2 for Kubernetes.*/def commitBatchChunked(fileBatches: Vector[FileBatch],@@ -187,9 +198,9 @@ class BulkWriteApi(client: ArangoClient) {val chunks = chunkForCommit(fileBatches, chunkSize, maxPayloadBytes)val multiChunk = chunks.size > 1 && baseRev > 0+ val finalRev = baseRev + chunks.size// Pre-tombstone: collect all entity IDs upfront and retire/tombstone in one parallel pass.- // Uses baseRev+1 as the tombstone rev, which is the same rev chunk 1 will use for its inserts.val preTombstoneIO: IO[Unit] =if (!multiChunk) IO.unitelse {@@ -208,11 +219,13 @@ class BulkWriteApi(client: ArangoClient) {for {_ <- preTombstoneIO- finalRev <- chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>- revIO.flatMap { currentRev =>- commitBatch(chunk, currentRev, skipTombstone = multiChunk).map(_.newRev.value)- }+ // Pre-assign revisions and run all chunks concurrently.+ // Each chunk writes to unique keys (logicalId_rev) so no conflicts.+ _ <- chunks.zipWithIndex.toVector.parTraverse { case (chunk, idx) =>+ commitBatch(chunk, baseRev + idx, skipTombstone = multiChunk)}+ // Update revision counter once at the end.+ _ <- updateRevision(finalRev)} yield CommitResult(Rev(finalRev), CommitStatus.Ok)}