Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
perf(ingest): strip chunks/claims in map mode and increase bulk write limits
Tanner Kocherdd2c0a8c7fda
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex d658c126..22930e64 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -202,6 +202,19 @@ function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {};}+/** Strip chunks and claims from patches — mapMode only needs nodes + edges. */+function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {+ return {+ ...patch,+ ops: patch.ops.filter(op => {+ if (op.type === 'AssertClaim' || op.type === 'RetractClaim') return false;+ if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;+ if (op.type === 'UpsertEdge' && (op['predicate'] === 'CONTAINS_CHUNK' || op['predicate'] === 'NEXT')) return false;+ return true;+ }),+ };+}+const COMMIT_CONFLICT_RETRY_PATTERNS = ['write-write conflict','timeout waiting to lock key',@@ -614,9 +627,11 @@ export async function ingestFiles(patchesApplied++;} catch (commitErr) {parseErrors++;- const errMsg = String(commitErr);- const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;- process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${truncated}\n`);+ if (debug) {+ const errMsg = String(commitErr);+ const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;+ process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${truncated}\n`);+ }}}} finally {@@ -676,7 +691,8 @@ export async function ingestFiles(for (let j = 0; j < batch.length; j++) {const { parsed: p, hash, previousHash } = batch[j];try {- const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);+ let patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);+ if (mapMode) patch = stripMapModeOps(patch);preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));} catch (err) {parseErrors++;@@ -741,7 +757,8 @@ export async function ingestFiles(for (let j = 0; j < allParsed.length; j++) {const { parsed: p, hash, previousHash } = allParsed[j];try {- const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);+ let patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);+ if (mapMode) patch = stripMapModeOps(patch);preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));} catch (err) {parseErrors++;diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 7c2e64ba..711ca552 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -23,8 +23,8 @@ class BulkWriteApi(client: ArangoClient) {private type BulkDoc = java.util.Map[String, AnyRef]private val CurrentRevisionKey = "current"private val SourceGraphRevisionKey = "source-graph"- private val DefaultChunkSize = 100- private val DefaultChunkPayloadBytes = 4 * 1024 * 1024+ private val DefaultChunkSize = 500+ private val DefaultChunkPayloadBytes = 16 * 1024 * 1024private val MaxNodeDocs = 10000private val MaxEdgeDocs = 20000private val MaxClaimDocs = 10000