Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
perf(db): parallelize bulk insert batches, raise connection pool, drop waitForSync
Tanner Kocher928f5c321747
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scalaindex 2e8d7451..fbc51a3b 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala@@ -238,7 +238,7 @@ object ArangoClient {.host(host, port).user(user).password(if (password.isEmpty) null else password)- .maxConnections(16)+ .maxConnections(32).build()val existingDbs = arango.getDatabases.asScala.toSetif (!existingDbs.contains(database)) {diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scalaindex 5900f8fa..715e214c 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala@@ -157,7 +157,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {.getOrElse(logicalId)val tombstoneAql = """UPDATE { _key: @key } WITH { deleted_rev: @rev, updated_at: @now } IN nodes- OPTIONS { waitForSync: true }+ OPTIONS { waitForSync: false }"""val newKey = s"${logicalId}_${newRev}"val upsertAql = """@@ -173,7 +173,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {attrs: @attrs, provenance: @provenance,deleted_rev: null, updated_at: @now}- IN nodes OPTIONS { waitForSync: true }+ IN nodes OPTIONS { waitForSync: false }"""for {_ <- client.execute(tombstoneAql, Map(@@ -211,7 +211,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {attrs: @attrs, provenance: @provenance,deleted_rev: null, updated_at: @now}- IN nodes OPTIONS { waitForSync: true }+ IN nodes OPTIONS { waitForSync: false }"""client.execute(upsertAql, Map("key" -> newKey.asInstanceOf[AnyRef],diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 711ca552..84f3d53a 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -5,6 +5,7 @@ import java.time.Instantimport java.util.UUIDimport cats.effect.IO+import cats.syntax.parallel._import cats.syntax.traverse._import io.circe.Jsonimport io.circe.syntax._@@ -50,16 +51,15 @@ class BulkWriteApi(client: ArangoClient) {)(insert: Seq[BulkDoc] => IO[Int]): IO[Long] =if (docs.isEmpty) IO.pure(0L)else {- val batches = chunkDocs(docs, maxDocs)- batches.foldLeft(IO.pure(0L)) { case (accIO, batch) =>+ val batches = chunkDocs(docs, maxDocs).toVector+ batches.parTraverse { batch =>for {- acc <- accIO- ms <- timedUnit(withRetry(insert(batch)))- _ <- if (debugBulkWrites)- logger.info(s"[bulk-write] collection=$collection docs=${batch.size} insertMs=$ms")- else IO.unit- } yield acc + ms- }+ ms <- timedUnit(withRetry(insert(batch)))+ _ <- if (debugBulkWrites)+ logger.info(s"[bulk-write] collection=$collection docs=${batch.size} insertMs=$ms")+ else IO.unit+ } yield ms+ }.map(_.sum)}def commitBatch(