Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(db): parallel chunk processing and compute-pool doc building

Tanner Kocher67168f0eded2
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 84f3d53a..b06736a7 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -62,6 +62,11 @@ class BulkWriteApi(client: ArangoClient) {
}.map(_.sum)
}
+ /**
+ * Build documents for a single chunk and insert them into ArangoDB.
+ * Document building runs on the compute pool (IO.delay) in parallel
+ * across FileBatches, then inserts run concurrently across collections.
+ */
def commitBatch(
fileBatches: Vector[FileBatch],
baseRev: Long,
@@ -70,20 +75,25 @@ class BulkWriteApi(client: ArangoClient) {
val newRev = baseRev + 1L
for {
- // Build all document maps in a single IO.blocking pass — one traversal instead of four.
+ // Build documents in parallel across FileBatches on the compute pool.
+ // Each FileBatch produces its own (nodes, edges, claims, patch) tuple independently.
docsAndBuildMs <- timed {
- IO.blocking {
- val nodesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
- val edgesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
- val claimsBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
- val patchesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
- fileBatches.foreach { fb =>
- nodesBuf ++= fb.nodeDocuments(newRev)
- edgesBuf ++= fb.edgeDocuments(newRev)
- claimsBuf ++= fb.claimDocuments(newRev)
- patchesBuf += fb.patchDocument(newRev)
- }
- (nodesBuf.result(), edgesBuf.result(), claimsBuf.result(), patchesBuf.result())
+ fileBatches.parTraverse { fb =>
+ IO.delay {
+ (fb.nodeDocuments(newRev), fb.edgeDocuments(newRev), fb.claimDocuments(newRev), fb.patchDocument(newRev))
+ }
+ }.map { results =>
+ val nodesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val edgesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val claimsBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val patchesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ results.foreach { case (nodes, edges, claims, patch) =>
+ nodesBuf ++= nodes
+ edgesBuf ++= edges
+ claimsBuf ++= claims
+ patchesBuf += patch
+ }
+ (nodesBuf.result(), edgesBuf.result(), claimsBuf.result(), patchesBuf.result())
}
}
(docs, buildDocsMs) = docsAndBuildMs
@@ -105,8 +115,7 @@ class BulkWriteApi(client: ArangoClient) {
.both(insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs)))
.map { case (((n, e), c), p) => (n, e, c, p) }
(insertNodesMs, insertEdgesMs, insertClaimsMs, insertPatchesMs) = allInsertMs
- updateRevMs <- timedUnit(updateRevision(newRev))
- totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs + updateRevMs
+ totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs
_ <- logChunkTiming(
s"rev=$newRev $chunkSummary",
totalMs,
@@ -118,7 +127,6 @@ class BulkWriteApi(client: ArangoClient) {
"insertEdges" -> insertEdgesMs,
"insertClaims" -> insertClaimsMs,
"insertPatches" -> insertPatchesMs,
- "updateRevision" -> updateRevMs,
)
)
} yield CommitResult(Rev(newRev), CommitStatus.Ok)
@@ -169,12 +177,15 @@ class BulkWriteApi(client: ArangoClient) {
)
/**
- * Commit file batches in chunks. Each chunk gets its own revision.
- * If a chunk fails, prior chunks are preserved (partial progress).
+ * Commit file batches in chunks with pre-assigned revisions.
+ *
+ * Each chunk gets a unique revision assigned upfront (baseRev+1, baseRev+2, ...),
+ * allowing all chunks to build documents and insert concurrently rather than
+ * waiting for the previous chunk's revision to be committed. The revision counter
+ * is updated once at the end to the final value.
*
* When multiple chunks exist and baseRev > 0 (incremental update), tombstone+retire
* runs once upfront for all files in parallel rather than once per chunk sequentially.
- * This reduces 2*N AQL UPDATE scans to 2 — e.g. 160 → 2 for Kubernetes.
*/
def commitBatchChunked(
fileBatches: Vector[FileBatch],
@@ -187,9 +198,9 @@ class BulkWriteApi(client: ArangoClient) {
val chunks = chunkForCommit(fileBatches, chunkSize, maxPayloadBytes)
val multiChunk = chunks.size > 1 && baseRev > 0
+ val finalRev = baseRev + chunks.size
// Pre-tombstone: collect all entity IDs upfront and retire/tombstone in one parallel pass.
- // Uses baseRev+1 as the tombstone rev, which is the same rev chunk 1 will use for its inserts.
val preTombstoneIO: IO[Unit] =
if (!multiChunk) IO.unit
else {
@@ -208,11 +219,13 @@ class BulkWriteApi(client: ArangoClient) {
for {
_ <- preTombstoneIO
- finalRev <- chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>
- revIO.flatMap { currentRev =>
- commitBatch(chunk, currentRev, skipTombstone = multiChunk).map(_.newRev.value)
- }
+ // Pre-assign revisions and run all chunks concurrently.
+ // Each chunk writes to unique keys (logicalId_rev) so no conflicts.
+ _ <- chunks.zipWithIndex.toVector.parTraverse { case (chunk, idx) =>
+ commitBatch(chunk, baseRev + idx, skipTombstone = multiChunk)
}
+ // Update revision counter once at the end.
+ _ <- updateRevision(finalRev)
} yield CommitResult(Rev(finalRev), CommitStatus.Ok)
}