Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf: reduce network round-trips and DB write overhead

Tanner Kocher7a6d9f541b60
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 22930e64..d2673921 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -540,7 +540,7 @@ export async function ingestFiles(
const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
- const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend
+ const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 400); // files per HTTP request to the backend
const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
const YIELD_EVERY = 100; // yield event loop every N files during parse
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
index fbc51a3b..b583c0c1 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
@@ -238,7 +238,7 @@ object ArangoClient {
.host(host, port)
.user(user)
.password(if (password.isEmpty) null else password)
- .maxConnections(32)
+ .maxConnections(64)
.build()
val existingDbs = arango.getDatabases.asScala.toSet
if (!existingDbs.contains(database)) {
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index b06736a7..6c3a161a 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -29,7 +29,7 @@ class BulkWriteApi(client: ArangoClient) {
private val MaxNodeDocs = 10000
private val MaxEdgeDocs = 20000
private val MaxClaimDocs = 10000
- private val MaxPatchDocs = 250
+ private val MaxPatchDocs = 500
private val SlowChunkMs = 2000L
private val debugBulkWrites = sys.env.get("IX_DEBUG").exists(v => v.nonEmpty && v != "0" && !v.equalsIgnoreCase("false"))
private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.bulk-write")
@@ -326,11 +326,18 @@ case class FileBatch(
result
}
- private def nodeKindToString(nk: NodeKind): String =
- nk.asJson.asString.getOrElse(nk.toString)
-
- lazy val estimatedPayloadBytes: Int =
- patch.asJson.noSpaces.getBytes(StandardCharsets.UTF_8).length + 1024
+ /** Heuristic payload estimate — avoids full Circe serialization round-trip.
+ * ~200 bytes per node op, ~180 per edge op, ~150 per claim op, +1KB overhead. */
+ lazy val estimatedPayloadBytes: Int = {
+ var nodes = 0; var edges = 0; var claims = 0
+ patch.ops.foreach {
+ case _: PatchOp.UpsertNode => nodes += 1
+ case _: PatchOp.UpsertEdge => edges += 1
+ case _: PatchOp.AssertClaim => claims += 1
+ case _ => ()
+ }
+ nodes * 200 + edges * 180 + claims * 150 + 1024
+ }
def nodeDocuments(rev: Long): Vector[java.util.Map[String, AnyRef]] = {
val now = Instant.now().toString
@@ -340,7 +347,7 @@ case class FileBatch(
doc.put("_key", s"${logicalId}_${rev}")
doc.put("logical_id", logicalId)
doc.put("id", logicalId)
- doc.put("kind", nodeKindToString(kind))
+ doc.put("kind", NodeKind.toWireString(kind))
doc.put("name", name)
doc.put("attrs", attrsToJavaMap(attrs))
doc.put("provenance", provenance)
diff --git a/memory-layer/src/main/scala/ix/memory/model/Node.scala b/memory-layer/src/main/scala/ix/memory/model/Node.scala
index ed35833f..49f449e2 100644
--- a/memory-layer/src/main/scala/ix/memory/model/Node.scala
+++ b/memory-layer/src/main/scala/ix/memory/model/Node.scala
@@ -86,6 +86,13 @@ object NodeKind {
implicit val decoder: Decoder[NodeKind] = Decoder[String].emap { s =>
nameMap.get(s).toRight(s"Unknown NodeKind: $s")
}
+
+ /** Pre-computed reverse lookup — avoids per-call Circe encoding in hot paths. */
+ private val kindToString: Map[NodeKind, String] = nameMap.map(_.swap)
+
+ /** Fast wire-format string without Circe encoding overhead. */
+ def toWireString(nk: NodeKind): String =
+ kindToString.getOrElse(nk, nk.toString)
}
final case class GraphNode(