Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: chunked DB writes for large batches (partial progress on failure)
riley02270498ca3b2f76
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 3c8fc0e4..865ba853 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -83,6 +83,27 @@ class BulkWriteApi(client: ArangoClient) {}}+ /**+ * Commit file batches in chunks. Each chunk gets its own revision.+ * If a chunk fails, prior chunks are preserved (partial progress).+ */+ def commitBatchChunked(+ fileBatches: Vector[FileBatch],+ baseRev: Long,+ chunkSize: Int = 100+ ): IO[CommitResult] = {+ if (fileBatches.isEmpty)+ return IO.pure(CommitResult(Rev(baseRev), CommitStatus.Ok))++ val chunks = fileBatches.grouped(chunkSize).toVector++ chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>+ revIO.flatMap { currentRev =>+ commitBatch(chunk, currentRev).map(_.newRev.value)+ }+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))+ }+private def updateRevision(newRev: Long): IO[Unit] =client.execute("""UPSERT { _key: @key }diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalaindex 5a85c93c..9558ddd1 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -60,7 +60,7 @@ class BulkIngestionService(skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCountlatestRev <- queryApi.getLatestRevresult <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))- else bulkWriteApi.commitBatch(validBatches.toVector, latestRev.value)+ else bulkWriteApi.commitBatchChunked(validBatches.toVector, latestRev.value)} yield IngestionResult(filesProcessed = files.size,patchesApplied = validBatches.size,