Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(db): parallelize bulk insert batches, raise connection pool, drop waitForSync

Tanner Kocher928f5c321747
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
index 2e8d7451..fbc51a3b 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
@@ -238,7 +238,7 @@ object ArangoClient {
.host(host, port)
.user(user)
.password(if (password.isEmpty) null else password)
- .maxConnections(16)
+ .maxConnections(32)
.build()
val existingDbs = arango.getDatabases.asScala.toSet
if (!existingDbs.contains(database)) {
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala
index 5900f8fa..715e214c 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphWriteApi.scala
@@ -157,7 +157,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {
.getOrElse(logicalId)
val tombstoneAql = """
UPDATE { _key: @key } WITH { deleted_rev: @rev, updated_at: @now } IN nodes
- OPTIONS { waitForSync: true }
+ OPTIONS { waitForSync: false }
"""
val newKey = s"${logicalId}_${newRev}"
val upsertAql = """
@@ -173,7 +173,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {
attrs: @attrs, provenance: @provenance,
deleted_rev: null, updated_at: @now
}
- IN nodes OPTIONS { waitForSync: true }
+ IN nodes OPTIONS { waitForSync: false }
"""
for {
_ <- client.execute(tombstoneAql, Map(
@@ -211,7 +211,7 @@ class ArangoGraphWriteApi(client: ArangoClient) extends GraphWriteApi {
attrs: @attrs, provenance: @provenance,
deleted_rev: null, updated_at: @now
}
- IN nodes OPTIONS { waitForSync: true }
+ IN nodes OPTIONS { waitForSync: false }
"""
client.execute(upsertAql, Map(
"key" -> newKey.asInstanceOf[AnyRef],
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 711ca552..84f3d53a 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -5,6 +5,7 @@ import java.time.Instant
import java.util.UUID
import cats.effect.IO
+import cats.syntax.parallel._
import cats.syntax.traverse._
import io.circe.Json
import io.circe.syntax._
@@ -50,16 +51,15 @@ class BulkWriteApi(client: ArangoClient) {
)(insert: Seq[BulkDoc] => IO[Int]): IO[Long] =
if (docs.isEmpty) IO.pure(0L)
else {
- val batches = chunkDocs(docs, maxDocs)
- batches.foldLeft(IO.pure(0L)) { case (accIO, batch) =>
+ val batches = chunkDocs(docs, maxDocs).toVector
+ batches.parTraverse { batch =>
for {
- acc <- accIO
- ms <- timedUnit(withRetry(insert(batch)))
- _ <- if (debugBulkWrites)
- logger.info(s"[bulk-write] collection=$collection docs=${batch.size} insertMs=$ms")
- else IO.unit
- } yield acc + ms
- }
+ ms <- timedUnit(withRetry(insert(batch)))
+ _ <- if (debugBulkWrites)
+ logger.info(s"[bulk-write] collection=$collection docs=${batch.size} insertMs=$ms")
+ else IO.unit
+ } yield ms
+ }.map(_.sum)
}
def commitBatch(