Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: chunked DB writes for large batches (partial progress on failure)

riley02270498ca3b2f76
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 3c8fc0e4..865ba853 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -83,6 +83,27 @@ class BulkWriteApi(client: ArangoClient) {
}
}
+ /**
+ * Commit file batches in chunks. Each chunk gets its own revision.
+ * If a chunk fails, prior chunks are preserved (partial progress).
+ */
+ def commitBatchChunked(
+ fileBatches: Vector[FileBatch],
+ baseRev: Long,
+ chunkSize: Int = 100
+ ): IO[CommitResult] = {
+ if (fileBatches.isEmpty)
+ return IO.pure(CommitResult(Rev(baseRev), CommitStatus.Ok))
+
+ val chunks = fileBatches.grouped(chunkSize).toVector
+
+ chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>
+ revIO.flatMap { currentRev =>
+ commitBatch(chunk, currentRev).map(_.newRev.value)
+ }
+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))
+ }
+
private def updateRevision(newRev: Long): IO[Unit] =
client.execute(
"""UPSERT { _key: @key }
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
index 5a85c93c..9558ddd1 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -60,7 +60,7 @@ class BulkIngestionService(
skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount
latestRev <- queryApi.getLatestRev
result <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))
- else bulkWriteApi.commitBatch(validBatches.toVector, latestRev.value)
+ else bulkWriteApi.commitBatchChunked(validBatches.toVector, latestRev.value)
} yield IngestionResult(
filesProcessed = files.size,
patchesApplied = validBatches.size,