Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(ingest): strip chunks/claims in map mode and increase bulk write limits

Tanner Kocherdd2c0a8c7fda
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index d658c126..22930e64 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -202,6 +202,19 @@ function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {
};
}
+/** Strip chunks and claims from patches — mapMode only needs nodes + edges. */
+function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {
+ return {
+ ...patch,
+ ops: patch.ops.filter(op => {
+ if (op.type === 'AssertClaim' || op.type === 'RetractClaim') return false;
+ if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;
+ if (op.type === 'UpsertEdge' && (op['predicate'] === 'CONTAINS_CHUNK' || op['predicate'] === 'NEXT')) return false;
+ return true;
+ }),
+ };
+}
+
const COMMIT_CONFLICT_RETRY_PATTERNS = [
'write-write conflict',
'timeout waiting to lock key',
@@ -614,9 +627,11 @@ export async function ingestFiles(
patchesApplied++;
} catch (commitErr) {
parseErrors++;
- const errMsg = String(commitErr);
- const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;
- process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${truncated}\n`);
+ if (debug) {
+ const errMsg = String(commitErr);
+ const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;
+ process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${truncated}\n`);
+ }
}
}
} finally {
@@ -676,7 +691,8 @@ export async function ingestFiles(
for (let j = 0; j < batch.length; j++) {
const { parsed: p, hash, previousHash } = batch[j];
try {
- const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
+ let patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
+ if (mapMode) patch = stripMapModeOps(patch);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
@@ -741,7 +757,8 @@ export async function ingestFiles(
for (let j = 0; j < allParsed.length; j++) {
const { parsed: p, hash, previousHash } = allParsed[j];
try {
- const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
+ let patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
+ if (mapMode) patch = stripMapModeOps(patch);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 7c2e64ba..711ca552 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -23,8 +23,8 @@ class BulkWriteApi(client: ArangoClient) {
private type BulkDoc = java.util.Map[String, AnyRef]
private val CurrentRevisionKey = "current"
private val SourceGraphRevisionKey = "source-graph"
- private val DefaultChunkSize = 100
- private val DefaultChunkPayloadBytes = 4 * 1024 * 1024
+ private val DefaultChunkSize = 500
+ private val DefaultChunkPayloadBytes = 16 * 1024 * 1024
private val MaxNodeDocs = 10000
private val MaxEdgeDocs = 20000
private val MaxClaimDocs = 10000