Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: speed up of ix map increasing speed for larger repos and allowed mapping for larger repos
kagebinarye7a95ea1202f
diff --git a/core-ingestion/dist/index.d.ts b/core-ingestion/dist/index.d.tsindex d24611fc..9eab1f0d 100644--- a/core-ingestion/dist/index.d.ts+++ b/core-ingestion/dist/index.d.ts@@ -40,7 +40,9 @@ export interface FileParseResult {}/** Returns true if a grammar is installed for the given file's language. */export declare function isGrammarSupported(filePath: string): boolean;-export declare function parseFile(filePath: string, source: string): FileParseResult | null;+export declare function parseFile(filePath: string, source: string, opts?: {+ emitChunks?: boolean;+}): FileParseResult | null;export interface ResolvedCallEdge {callerFilePath: string;callerName: string;@@ -85,8 +87,7 @@ export interface ResolvedEdge {* - Unambiguous class method: dstQualifiedKey === 'ClassName.method'* - Ambiguous (two entities share the same plain name): edge not emitted*/-/** Stats collected during resolution (exposed for debug instrumentation). */-export interface ResolveStats {+export declare function resolveEdges(results: FileParseResult[], stats?: {importLookups: number;transitiveLookups: number;globalFallbacks: number;@@ -97,8 +98,6 @@ export interface ResolveStats {resolvedQualifier: number;skippedSameFile: number;skippedAmbiguous: number;-}-export declare function resolveEdges(results: FileParseResult[]): ResolvedEdge[];-export declare function resolveEdges(results: FileParseResult[], statsOut: ResolveStats): ResolvedEdge[];+}): ResolvedEdge[];/** @deprecated Use resolveEdges instead. */export declare function resolveCallEdges(results: FileParseResult[]): ResolvedCallEdge[];diff --git a/core-ingestion/dist/index.js b/core-ingestion/dist/index.jsindex 33d01b76..dabe5504 100644--- a/core-ingestion/dist/index.js+++ b/core-ingestion/dist/index.js@@ -228,7 +228,7 @@ function unwrapRustCfgMacros(source) {// ---------------------------------------------------------------------------// Main parse function// ----------------------------------------------------------------------------export function parseFile(filePath, source) {+export function parseFile(filePath, source, opts) {const language = languageFromPath(filePath);if (!language)return null;@@ -548,27 +548,29 @@ export function parseFile(filePath, source) {continue;}}- for (const pendingChunk of pendingChunks) {- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);- chunks.push({- ...pendingChunk,- contentHash,- });- }- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file- if (chunks.length === 0) {- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);- chunks.push({- name: null,- chunkKind: 'file_body',- lineStart: 1,- lineEnd: sourceLineCount,- startByte: 0,- endByte: source.length,- contentHash,- language,- });+ if (opts?.emitChunks !== false) {+ for (const pendingChunk of pendingChunks) {+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);+ chunks.push({+ ...pendingChunk,+ contentHash,+ });+ }+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file+ if (chunks.length === 0) {+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);+ chunks.push({+ name: null,+ chunkKind: 'file_body',+ lineStart: 1,+ lineEnd: sourceLineCount,+ startByte: 0,+ endByte: source.length,+ contentHash,+ language,+ });+ }}return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };}diff --git a/core-ingestion/dist/patch-builder.d.ts b/core-ingestion/dist/patch-builder.d.tsindex 56c3ae9b..f7511c4b 100644--- a/core-ingestion/dist/patch-builder.d.ts+++ b/core-ingestion/dist/patch-builder.d.ts@@ -4,4 +4,6 @@ export declare function extractorName(): string;/** Previous extractor versions — their patches are superseded when re-ingesting. */export declare const PREVIOUS_EXTRACTORS: string[];export declare function buildPatch(result: FileParseResult, sourceHash: string, previousSourceHash?: string): GraphPatchPayload;-export declare function buildPatchWithResolution(result: FileParseResult, sourceHash: string, resolvedEdges: ResolvedEdge[], previousSourceHash?: string): GraphPatchPayload;+export declare function buildPatchWithResolution(result: FileParseResult, sourceHash: string, resolvedEdges: ResolvedEdge[], previousSourceHash?: string, opts?: {+ emitClaims?: boolean;+}): GraphPatchPayload;diff --git a/core-ingestion/dist/patch-builder.js b/core-ingestion/dist/patch-builder.jsindex 026cba0f..0fc438dd 100644--- a/core-ingestion/dist/patch-builder.js+++ b/core-ingestion/dist/patch-builder.js@@ -239,7 +239,7 @@ export function buildPatch(result, sourceHash, previousSourceHash) {// buildPatchWithResolution — like buildPatch but fixes CALLS edge dst to point// to the actual defining file for cross-file calls resolved by resolveCallEdges.// ----------------------------------------------------------------------------export function buildPatchWithResolution(result, sourceHash, resolvedEdges, previousSourceHash) {+export function buildPatchWithResolution(result, sourceHash, resolvedEdges, previousSourceHash, opts) {// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }// Callers should pass only edges for this file (pre-grouped) for best performance,// but we still tolerate the full array for backward compatibility.@@ -381,15 +381,17 @@ export function buildPatchWithResolution(result, sourceHash, resolvedEdges, prevattrs: {},});}- for (const r of relationships) {- const srcKey = resolveKey(r.srcName);- ops.push({- type: 'AssertClaim',- entityId: nodeId(filePath, srcKey),- field: `${r.predicate.toLowerCase()}:${r.dstName}`,- value: r.dstName,- confidence: null,- });+ if (opts?.emitClaims !== false) {+ for (const r of relationships) {+ const srcKey = resolveKey(r.srcName);+ ops.push({+ type: 'AssertClaim',+ entityId: nodeId(filePath, srcKey),+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,+ value: r.dstName,+ confidence: null,+ });+ }}const extractor = extractorName();const patchId = computePatchId(filePath, sourceHash, extractor);diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex dd12532c..78f05b10 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -283,7 +283,7 @@ function unwrapRustCfgMacros(source: string): string {// Main parse function// ----------------------------------------------------------------------------export function parseFile(filePath: string, source: string): FileParseResult | null {+export function parseFile(filePath: string, source: string, opts?: { emitChunks?: boolean }): FileParseResult | null {const language = languageFromPath(filePath);if (!language) return null;@@ -629,28 +629,30 @@ export function parseFile(filePath: string, source: string): FileParseResult | n}}- for (const pendingChunk of pendingChunks) {- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);- chunks.push({- ...pendingChunk,- contentHash,- });- }+ if (opts?.emitChunks !== false) {+ for (const pendingChunk of pendingChunks) {+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);+ chunks.push({+ ...pendingChunk,+ contentHash,+ });+ }- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file- if (chunks.length === 0) {- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);- chunks.push({- name: null,- chunkKind: 'file_body',- lineStart: 1,- lineEnd: sourceLineCount,- startByte: 0,- endByte: source.length,- contentHash,- language,- });+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file+ if (chunks.length === 0) {+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);+ chunks.push({+ name: null,+ chunkKind: 'file_body',+ lineStart: 1,+ lineEnd: sourceLineCount,+ startByte: 0,+ endByte: source.length,+ contentHash,+ language,+ });+ }}return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.tsindex df704395..5500cb24 100644--- a/core-ingestion/src/patch-builder.ts+++ b/core-ingestion/src/patch-builder.ts@@ -272,6 +272,7 @@ export function buildPatchWithResolution(sourceHash: string,resolvedEdges: ResolvedEdge[],previousSourceHash?: string,+ opts?: { emitClaims?: boolean },): GraphPatchPayload {// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }// Callers should pass only edges for this file (pre-grouped) for best performance,@@ -420,15 +421,17 @@ export function buildPatchWithResolution(});}- for (const r of relationships) {- const srcKey = resolveKey(r.srcName);- ops.push({- type: 'AssertClaim',- entityId: nodeId(filePath, srcKey),- field: `${r.predicate.toLowerCase()}:${r.dstName}`,- value: r.dstName,- confidence: null,- });+ if (opts?.emitClaims !== false) {+ for (const r of relationships) {+ const srcKey = resolveKey(r.srcName);+ ops.push({+ type: 'AssertClaim',+ entityId: nodeId(filePath, srcKey),+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,+ value: r.dstName,+ confidence: null,+ });+ }}const extractor = extractorName();diff --git a/ix-cli/src/cli/commands/explain.ts b/ix-cli/src/cli/commands/explain.tsindex f2663338..af5fbc0d 100644--- a/ix-cli/src/cli/commands/explain.ts+++ b/ix-cli/src/cli/commands/explain.ts@@ -52,7 +52,7 @@ export function registerExplainCommand(program: Command): void {console.log(JSON.stringify(output, null, 2));} else {if (facts.stale) {- renderWarning("Source has changed since last ingest. Run ix ingest to update.");+ renderWarning("Source has changed since last ingest. Run ix map to update.");}renderSection("Explanation");@@ -213,7 +213,7 @@ async function rawExplain(}console.log(JSON.stringify(output, null, 2));} else {- if (stale) renderWarning("Source has changed since last ingest. Run ix ingest to update.");+ if (stale) renderWarning("Source has changed since last ingest. Run ix map to update.");formatExplain(result, "text");}}diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex d62f208f..3f809146 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -200,8 +200,20 @@ function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {};}-function isWriteConflict(err: unknown): boolean {- return String(err).includes('write-write conflict');+const COMMIT_CONFLICT_RETRY_PATTERNS = [+ 'write-write conflict',+ 'timeout waiting to lock key',+ 'error: 1200',+ // Transport-level failures (k8s ingress reset, socket drop under load).+ // Safe to retry because the server never received / committed the payload.+ 'fetch failed',+ 'econnreset',+ 'econnrefused',+];++export function isRetryableCommitConflict(err: unknown): boolean {+ const message = String(err).toLowerCase();+ return COMMIT_CONFLICT_RETRY_PATTERNS.some(pattern => message.includes(pattern));}async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Promise<T> {@@ -209,9 +221,9 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Protry {return await fn();} catch (err) {- if (attempt >= maxRetries || !isWriteConflict(err)) throw err;- // ArangoDB write-write conflicts resolve as soon as the competing transaction- // commits — typically <10ms. Use short jittered delay to avoid thundering herd.+ if (attempt >= maxRetries || !isRetryableCommitConflict(err)) throw err;+ // ArangoDB lock conflicts usually clear as soon as the competing transaction+ // commits. Transport errors may need a bit more breathing room under k8s load.const delay = 10 * (1 << attempt) + Math.random() * 10;await new Promise<void>(r => setTimeout(r, delay));}@@ -339,13 +351,6 @@ export async function ingestFiles(let progressStart = performance.now();const interval = opts.format === 'text' ? setInterval(() => {- const elapsed = performance.now() - progressStart;- const tau = progressPhase === 'Saving' ? 3000 : 2000;- let display = progressCurrent;- if (progressTotal > 0 && progressCurrent < progressTotal) {- const simulated = Math.floor((1 - Math.exp(-elapsed / tau)) * 0.88 * progressTotal);- display = Math.max(display, simulated);- }if (debug && currentWorkLabel) {const workElapsed = performance.now() - currentWorkStart;if (workElapsed >= SLOW_WORK_LOG_MS && (lastSlowWorkNotice === 0 || workElapsed - lastSlowWorkNotice >= SLOW_WORK_REPEAT_MS)) {@@ -353,7 +358,7 @@ export async function ingestFiles(process.stderr.write(`\n [slow ${progressPhase.toLowerCase()}] ${currentWorkLabel} (${Math.round(workElapsed / 1000)}s)\n`);}}- process.stderr.write('\r' + renderProgressLine(progressPhase, display, progressTotal));+ process.stderr.write('\r' + renderProgressLine(progressPhase, progressCurrent, progressTotal));}, 80) : null;let filesDiscovered = 0;@@ -376,21 +381,41 @@ export async function ingestFiles(};// Phase timing marks- const timings = { moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0, resolveMs: 0, buildPatchMs: 0, commitMs: 0 };+ const timings = {+ moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,+ parseOnlyMs: 0, // sum of just parseFile() call durations+ resolveMs: 0, buildPatchMs: 0, commitMs: 0,+ stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)+ bulkCommitMs: 0, // time in successful commitPatchBulk() calls+ fallbackCommitMs: 0, // time in per-file fallback commitPatch() calls+ };const resolveStats = {importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,resolvedGlobal: 0, resolvedQualifier: 0, skippedSameFile: 0, skippedAmbiguous: 0,};+ // Per-language parse timing: language → { ms, files }+ const langParseMs = new Map<string, { ms: number; files: number }>();+ // Total graph op counts across all committed batches+ let totalNodesEmitted = 0;+ let totalEdgesEmitted = 0;+ let totalClaimsEmitted = 0;+const logSaveTiming = (files: number,resolveEdgesMs: number,buildPatchMs: number,+ stripMs: number,commitMs: number,totalSaveMs: number,+ nodes: number,+ edges: number,+ claims: number,): void => {process.stderr.write(- `\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs} commitMs=${commitMs} totalSaveMs=${totalSaveMs}\n`+ `\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` ++ (mapMode ? ` stripChunkOpsMs=${stripMs}` : '') ++ ` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`);};@@ -492,7 +517,8 @@ export async function ingestFiles(const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large reposconst SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming aboveconst COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend- const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 4); // parallel HTTP save requests+ const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', mapMode ? 2 : 4); // parallel HTTP save requests+ const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflictsconst YIELD_EVERY = 100; // yield event loop every N files during parseif (debug) {@@ -532,6 +558,9 @@ export async function ingestFiles(const commitMsPerChunk = new Array<number>(chunks.length).fill(0);let nextChunk = 0;+ // Fallback mutex: serializes per-file commits across all workers so that+ // concurrent fallback loops don't race on revisions.current (Error 1200).+ let fallbackTail: Promise<void> = Promise.resolve();const runChunk = async (ci: number): Promise<void> => {const chunk = chunks[ci];@@ -543,8 +572,10 @@ export async function ingestFiles(try {setCurrentWork(`commit ${startFile}-${endFile} of ${totalFiles} ending ${endingPath}`);const commitStart = performance.now();- const result = await retryOnConflict(() => client.commitPatchBulk(patches), 3);- commitMsPerChunk[ci] = Math.round(performance.now() - commitStart);+ const result = await retryOnConflict(() => client.commitPatchBulk(patches), COMMIT_CONFLICT_RETRIES);+ const chunkMs = Math.round(performance.now() - commitStart);+ commitMsPerChunk[ci] = chunkMs;+ timings.bulkCommitMs += chunkMs;if (result.rev > latestRev) latestRev = result.rev;patchesApplied += patches.length;} catch (err) {@@ -553,17 +584,30 @@ export async function ingestFiles(`\n [bulk failed, falling back to per-file] files ${startFile}-${endFile} (${patches.length} patches): ${err}\n`);}- for (const item of chunk) {- try {- const commitStart = performance.now();- const result = await retryOnConflict(() => client.commitPatch(item.patch), 3);- commitMsPerChunk[ci] += Math.round(performance.now() - commitStart);- if (result.rev > latestRev) latestRev = result.rev;- patchesApplied++;- } catch (commitErr) {- parseErrors++;- process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${commitErr}\n`);+ // Chain this chunk's fallback work onto the shared tail so that only one+ // fallback loop runs at a time. This prevents concurrent commitPatch+ // transactions from racing on revisions.current (write-write conflict).+ const prev = fallbackTail;+ let resolveThis!: () => void;+ fallbackTail = new Promise<void>(r => { resolveThis = r; });+ await prev;+ try {+ for (const item of chunk) {+ try {+ const commitStart = performance.now();+ const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);+ const chunkMs = Math.round(performance.now() - commitStart);+ commitMsPerChunk[ci] += chunkMs;+ timings.fallbackCommitMs += chunkMs;+ if (result.rev > latestRev) latestRev = result.rev;+ patchesApplied++;+ } catch (commitErr) {+ parseErrors++;+ process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${commitErr}\n`);+ }}+ } finally {+ resolveThis();}}@@ -591,13 +635,15 @@ export async function ingestFiles(const flushBatch = async (batch: ParsedFile[]): Promise<void> => {if (batch.length === 0) return;filesChanged += batch.length;- const previousPhase = progressPhase;const saveStart = performance.now();let resolveEdgesMs = 0;let buildPatchMs = 0;+ let localStripMs = 0;let commitMs = 0;+ let batchNodes = 0;+ let batchEdges = 0;+ let batchClaims = 0;progressPhase = 'Saving';- progressStart = performance.now();try {setCurrentWork(`resolve+commit batch ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);@@ -617,8 +663,7 @@ export async function ingestFiles(for (let j = 0; j < batch.length; j++) {const { parsed: p, hash, previousHash } = batch[j];try {- const rawPatch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);- const patch = mapMode ? stripChunkOps(rawPatch) : rawPatch;+ const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));} catch (err) {parseErrors++;@@ -626,15 +671,26 @@ export async function ingestFiles(}}buildPatchMs += Math.round(performance.now() - buildStart);+ for (const pp of preparedPatches) {+ for (const op of pp.patch.ops) {+ if (op.type === 'UpsertNode') batchNodes++;+ else if (op.type === 'UpsertEdge') batchEdges++;+ else if (op.type === 'AssertClaim') batchClaims++;+ }+ }if (preparedPatches.length === 0) return;commitMs += await commitPreparedPatches(preparedPatches, batch.length);} finally {const totalSaveMs = Math.round(performance.now() - saveStart);timings.resolveMs += resolveEdgesMs;timings.buildPatchMs += buildPatchMs;+ timings.stripChunkOpsMs += localStripMs;timings.commitMs += commitMs;- logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs);- progressPhase = previousPhase;+ totalNodesEmitted += batchNodes;+ totalEdgesEmitted += batchEdges;+ totalClaimsEmitted += batchClaims;+ logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);+ progressPhase = 'Parsing';progressStart = performance.now();}};@@ -647,11 +703,14 @@ export async function ingestFiles(const saveStart = performance.now();let resolveEdgesMs = 0;let buildPatchMs = 0;+ let localStripMs = 0;let commitMs = 0;+ let batchNodes = 0;+ let batchEdges = 0;+ let batchClaims = 0;progressPhase = 'Saving';progressTotal = allParsed.length;progressCurrent = 0;- progressStart = performance.now();try {setCurrentWork(`resolve ${allParsed.length} files`);const resolveStart = performance.now();@@ -669,8 +728,7 @@ export async function ingestFiles(for (let j = 0; j < allParsed.length; j++) {const { parsed: p, hash, previousHash } = allParsed[j];try {- const rawPatch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);- const patch = mapMode ? stripChunkOps(rawPatch) : rawPatch;+ const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));} catch (err) {parseErrors++;@@ -678,6 +736,13 @@ export async function ingestFiles(}}buildPatchMs += Math.round(performance.now() - buildStart);+ for (const pp of preparedPatches) {+ for (const op of pp.patch.ops) {+ if (op.type === 'UpsertNode') batchNodes++;+ else if (op.type === 'UpsertEdge') batchEdges++;+ else if (op.type === 'AssertClaim') batchClaims++;+ }+ }if (preparedPatches.length === 0) {progressCurrent = allParsed.length;return;@@ -687,8 +752,12 @@ export async function ingestFiles(const totalSaveMs = Math.round(performance.now() - saveStart);timings.resolveMs += resolveEdgesMs;timings.buildPatchMs += buildPatchMs;+ timings.stripChunkOpsMs += localStripMs;timings.commitMs += commitMs;- logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs);+ totalNodesEmitted += batchNodes;+ totalEdgesEmitted += batchEdges;+ totalClaimsEmitted += batchClaims;+ logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);progressPhase = previousPhase;progressStart = performance.now();}@@ -732,8 +801,14 @@ export async function ingestFiles(if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);continue;}- const parsed = (ingestion.parseFile as Function)(filePath, sourceText);+ const p0 = performance.now();+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });+ const pfMs = performance.now() - p0;if (!parsed) { filesSkipped++; progressCurrent++; continue; }+ timings.parseOnlyMs += pfMs;+ const lang: string = (parsed as any).language ?? 'unknown';+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);entitiesParsed += parsed.entities.length;allParsed.push({ filePath, parsed, hash, previousHash });progressCurrent++;@@ -776,8 +851,14 @@ export async function ingestFiles(if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);continue;}- const parsed = (ingestion.parseFile as Function)(filePath, sourceText);+ const p0 = performance.now();+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });+ const pfMs = performance.now() - p0;if (!parsed) { filesSkipped++; progressCurrent++; continue; }+ timings.parseOnlyMs += pfMs;+ const lang: string = (parsed as any).language ?? 'unknown';+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);entitiesParsed += parsed.entities.length;const entry: ParsedFile = { filePath, parsed, hash, previousHash: previousHash !== hash ? previousHash : undefined };if (useParseAllFirst) { allParsedB.push(entry); } else { batch.push(entry); }@@ -833,7 +914,15 @@ export async function ingestFiles(latestRev,skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge, minifiedLikely },elapsedSeconds: parseFloat(elapsed),- timings,+ timings: {+ ...timings,+ parseOnlyMs: Math.round(timings.parseOnlyMs),+ stripChunkOpsMs: Math.round(timings.stripChunkOpsMs),+ },+ emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },+ langParseMs: Object.fromEntries(+ [...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])+ ),resolveStats,}, null, 2));} else if (opts.printSummary !== false) {@@ -855,13 +944,30 @@ export async function ingestFiles(if (debug) {console.log(chalk.dim(`\n Phase timings:`));- console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));- console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));- console.log(chalk.dim(` hash: ${timings.hashMs}ms`));- console.log(chalk.dim(` parse: ${timings.parseMs}ms`));- console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));- console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));- console.log(chalk.dim(` commit: ${timings.commitMs}ms`));+ console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));+ console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));+ console.log(chalk.dim(` hash: ${timings.hashMs}ms`));+ console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));+ console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));+ console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));+ console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));+ if (mapMode) {+ console.log(chalk.dim(` stripChunk: ${Math.round(timings.stripChunkOpsMs)}ms`));+ }+ console.log(chalk.dim(` commit: ${timings.commitMs}ms`));+ console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));+ console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));+ console.log(chalk.dim(`\n Emitted ops:`));+ console.log(chalk.dim(` nodes: ${totalNodesEmitted}`));+ console.log(chalk.dim(` edges: ${totalEdgesEmitted}`));+ console.log(chalk.dim(` claims: ${totalClaimsEmitted}`));+ if (langParseMs.size > 0) {+ console.log(chalk.dim(`\n Per-language parse times:`));+ const sorted = [...langParseMs.entries()].sort((a, b) => b[1].ms - a[1].ms);+ for (const [lang, stat] of sorted) {+ console.log(chalk.dim(` ${lang.padEnd(12)} ${Math.round(stat.ms)}ms (${stat.files} files)`));+ }+ }console.log(chalk.dim(`\n Resolve stats:`));console.log(chalk.dim(` import lookups: ${resolveStats.importLookups}`));console.log(chalk.dim(` transitive lookups: ${resolveStats.transitiveLookups}`));diff --git a/ix-cli/src/cli/commands/locate.ts b/ix-cli/src/cli/commands/locate.tsindex 0842a8f4..11096451 100644--- a/ix-cli/src/cli/commands/locate.ts+++ b/ix-cli/src/cli/commands/locate.ts@@ -219,7 +219,7 @@ function outputLocate(output: LocateOutput, symbol: string, format: string): voireturn;}- if (output.stale) renderWarning("Some results may be stale. Run ix ingest to update.");+ if (output.stale) renderWarning("Some results may be stale. Run ix map to update.");if (!output.resolvedTarget) {stderr(`No graph entity found for "${symbol}".`);diff --git a/ix-cli/src/cli/commands/read.ts b/ix-cli/src/cli/commands/read.tsindex 88c8b46a..d67f5864 100644--- a/ix-cli/src/cli/commands/read.ts+++ b/ix-cli/src/cli/commands/read.ts@@ -60,7 +60,7 @@ function outputResult(result: ReadResult, format: string): void {if (format === "json") {console.log(JSON.stringify(result, null, 2));} else {- if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix ingest to update.\n"));+ if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix map to update.\n"));if (result.targetType === "symbol" || result.targetType === "filename-match") {stderr(chalk.dim(` ${result.path}:${result.lineStart}-${result.lineEnd}\n`));}diff --git a/ix-cli/src/cli/commands/status.ts b/ix-cli/src/cli/commands/status.tsindex 5fd9490c..824419e2 100644--- a/ix-cli/src/cli/commands/status.ts+++ b/ix-cli/src/cli/commands/status.ts@@ -51,7 +51,7 @@ export function registerStatusCommand(program: Command): void {if (staleInfo.staleFiles > staleInfo.sampleChangedFiles.length) {renderNote(`... and ${staleInfo.staleFiles - staleInfo.sampleChangedFiles.length} more`);}- renderNote("Run ix ingest to update.");+ renderNote("Run ix map to update.");} else {renderSuccess("Graph is up to date.");}diff --git a/ix-cli/src/cli/explain/render.ts b/ix-cli/src/cli/explain/render.tsindex 3a9c2bed..52f6fd0c 100644--- a/ix-cli/src/cli/explain/render.ts+++ b/ix-cli/src/cli/explain/render.ts@@ -524,7 +524,7 @@ function renderNotes(facts: EntityFacts): string[] {for (const d of facts.diagnostics) {switch (d.code) {case "unresolved_call_target":- notes.push("Some downstream calls could not be resolved to named entities. Run `ix ingest` to improve coverage.");+ notes.push("Some downstream calls could not be resolved to named entities. Run `ix map` to improve coverage.");break;case "stale_source":notes.push("Source file has changed since last ingest — results may be incomplete.");diff --git a/ix-cli/src/cli/explain/subsystem.ts b/ix-cli/src/cli/explain/subsystem.tsindex 8fe19052..f7d9cd74 100644--- a/ix-cli/src/cli/explain/subsystem.ts+++ b/ix-cli/src/cli/explain/subsystem.ts@@ -311,7 +311,7 @@ function densityNote(chunkDensity: number): string {return " High chunk density indicates thorough code coverage in the knowledge graph.";}if (chunkDensity < 2.0) {- return " Low chunk density - run 'ix ingest' to improve coverage.";+ return " Low chunk density - run 'ix ingest' to enable ix read / ix explain on this subsystem.";}return "";}diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scalaindex a9f38cd4..ee377976 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala@@ -84,6 +84,7 @@ class ArangoClient private (db: ArangoDatabase) {val opts = new DocumentCreateOptions().overwriteMode(OverwriteMode.valueOf(overwriteMode)).waitForSync(false)+ .silent(true)val docs = new java.util.ArrayList[java.util.Map[String, AnyRef]](documents.size)documents.foreach(docs.add)val result = db.collection(collection).insertDocuments(docs, opts)@@ -243,7 +244,7 @@ object ArangoClient {.host(host, port).user(user).password(if (password.isEmpty) null else password)- .maxConnections(8)+ .maxConnections(16).build()val existingDbs = arango.getDatabases.asScala.toSetif (!existingDbs.contains(database)) {diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scalaindex 4a31b3e3..b6756095 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala@@ -308,7 +308,8 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {val nodeIdStr = entityId.value.toStringclient.query("""FOR p IN patches- | FILTER LENGTH(+ | FILTER @nodeId IN (IS_ARRAY(p.data.entityIds) ? p.data.entityIds : [])+ | OR LENGTH(| FOR op IN (IS_ARRAY(p.data.ops) ? p.data.ops : [])| FILTER op.id == @nodeId OR op.entityId == @nodeId| RETURN 1diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 3c92a0c9..7c2e64ba 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -96,10 +96,15 @@ class BulkWriteApi(client: ArangoClient) {else timedUnit(tombstoneExistingNodes(logicalIds, newRev)).both(timedUnit(retireOldClaims(fileBatches, newRev)))(tombstoneMs: Long, retireClaimsMs: Long) = tombstoneAndRetireMs- insertNodesMs <- insertCollectionChunked("nodes", allNodes, MaxNodeDocs)(docs => client.bulkInsert("nodes", docs))- insertEdgesMs <- insertCollectionChunked("edges", allEdges, MaxEdgeDocs)(docs => client.bulkInsertEdges("edges", docs))- insertClaimsMs <- insertCollectionChunked("claims", allClaims, MaxClaimDocs)(docs => client.bulkInsert("claims", docs))- insertPatchesMs <- insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs))+ // Run all four collection inserts concurrently — they write to independent collections+ // with no referential-integrity enforcement between them. Wall time drops from the+ // sum of all four to the max of any one (typically nodes or edges).+ allInsertMs <- insertCollectionChunked("nodes", allNodes, MaxNodeDocs)(docs => client.bulkInsert("nodes", docs))+ .both(insertCollectionChunked("edges", allEdges, MaxEdgeDocs)(docs => client.bulkInsertEdges("edges", docs)))+ .both(insertCollectionChunked("claims", allClaims, MaxClaimDocs)(docs => client.bulkInsert("claims", docs)))+ .both(insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs)))+ .map { case (((n, e), c), p) => (n, e, c, p) }+ (insertNodesMs, insertEdgesMs, insertClaimsMs, insertPatchesMs) = allInsertMsupdateRevMs <- timedUnit(updateRevision(newRev))totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs + updateRevMs_ <- logChunkTiming(@@ -372,11 +377,68 @@ case class FileBatch(}def patchDocument(rev: Long): java.util.Map[String, AnyRef] = {+ val source = new java.util.HashMap[String, AnyRef]()+ source.put("uri", patch.source.uri)+ source.put("sourceHash", patch.source.sourceHash.orNull)+ source.put("extractor", patch.source.extractor)+ source.put("sourceType", patch.source.sourceType.asJson.asString.getOrElse(patch.source.sourceType.toString))++ val replaces = new java.util.ArrayList[String](patch.replaces.size)+ patch.replaces.foreach(id => replaces.add(id.value.toString))++ val entityIds = new java.util.LinkedHashSet[String]()+ var nodeOpCount = 0+ var edgeOpCount = 0+ var claimOpCount = 0+ patch.ops.foreach {+ case PatchOp.UpsertNode(id, _, _, _) =>+ nodeOpCount += 1+ entityIds.add(id.value.toString)+ case PatchOp.DeleteNode(id) =>+ nodeOpCount += 1+ entityIds.add(id.value.toString)+ case PatchOp.UpsertEdge(id, _, _, _, _) =>+ edgeOpCount += 1+ entityIds.add(id.value.toString)+ case PatchOp.DeleteEdge(id) =>+ edgeOpCount += 1+ entityIds.add(id.value.toString)+ case PatchOp.AssertClaim(entityId, _, _, _, _) =>+ claimOpCount += 1+ entityIds.add(entityId.value.toString)+ case PatchOp.RetractClaim(claimId) =>+ claimOpCount += 1+ entityIds.add(claimId.value.toString)+ }++ val entityIdList = new java.util.ArrayList[String](entityIds.size)+ val entityIdIter = entityIds.iterator()+ while (entityIdIter.hasNext) {+ entityIdList.add(entityIdIter.next())+ }++ val data = new java.util.HashMap[String, AnyRef]()+ data.put("patchId", patch.patchId.value.toString)+ data.put("actor", patch.actor)+ data.put("timestamp", patch.timestamp.toString)+ data.put("source", source)+ data.put("baseRev", Long.box(patch.baseRev.value))+ data.put("replaces", replaces)+ data.put("intent", patch.intent.orNull)+ data.put("entityIds", entityIdList)+ data.put("opCount", Int.box(patch.ops.size))+ data.put("nodeOpCount", Int.box(nodeOpCount))+ data.put("edgeOpCount", Int.box(edgeOpCount))+ data.put("claimOpCount", Int.box(claimOpCount))+val doc = new java.util.HashMap[String, AnyRef]()doc.put("_key", patch.patchId.value.toString)doc.put("patch_id", patch.patchId.value.toString)doc.put("rev", Long.box(rev))- doc.put("data", jsonToJava(patch.asJson))+ // Bulk ingest only needs patch metadata plus touched-entity IDs for provenance+ // and source-hash lookups. Storing full ops here duplicates the graph payload+ // and significantly inflates Kubernetes save latency.+ doc.put("data", data)doc: java.util.Map[String, AnyRef]}}diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scalaindex 35ec4613..234fa82e 100644--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala@@ -221,6 +221,38 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD}}+ it should "return provenance chain for entity after bulk commit" in {+ clientResource.use { client =>+ val writeApi = new ArangoGraphWriteApi(client)+ val queryApi = new ArangoGraphQueryApi(client)+ val bulkWriteApi = new BulkWriteApi(client)+ val app = buildRoutes(client, writeApi, queryApi)+ val nodeId = NodeId(UUID.randomUUID())+ val patch = makePatch(ops = Vector(PatchOp.UpsertNode(nodeId, NodeKind.Function, "bulk_prov_func", Map.empty[String, Json])))+ val provenance = new java.util.HashMap[String, AnyRef]()+ provenance.put("source_uri", patch.source.uri)+ provenance.put("source_hash", patch.source.sourceHash.orNull)+ provenance.put("extractor", patch.source.extractor)+ provenance.put("source_type", patch.source.sourceType.asJson.asString.getOrElse("code"))+ provenance.put("observed_at", patch.timestamp.toString)++ for {+ _ <- client.ensureSchema()+ _ <- cleanDatabase(client)+ _ <- bulkWriteApi.commitBatchChunked(+ Vector(FileBatch(patch.source.uri, patch.source.sourceHash, patch, provenance)),+ baseRev = 0L+ )+ resp <- app.run(Request[IO](Method.POST, Uri.unsafeFromString(s"/v1/provenance/${nodeId.value}")))+ body <- resp.as[Json]+ } yield {+ resp.status shouldBe Status.Ok+ body.hcursor.downField("entityId").as[String] shouldBe Right(nodeId.value.toString)+ body.hcursor.downField("chain").focus.flatMap(_.asArray).exists(_.nonEmpty) shouldBe true+ }+ }+ }+it should "create a decision node via POST /v1/decide" in {clientResource.use { client =>val app = buildRoutes(client, new ArangoGraphWriteApi(client), new ArangoGraphQueryApi(client))