Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: speed up of ix map increasing speed for larger repos and allowed mapping for larger repos

kagebinarye7a95ea1202f
diff --git a/core-ingestion/dist/index.d.ts b/core-ingestion/dist/index.d.ts
index d24611fc..9eab1f0d 100644
--- a/core-ingestion/dist/index.d.ts
+++ b/core-ingestion/dist/index.d.ts
@@ -40,7 +40,9 @@ export interface FileParseResult {
}
/** Returns true if a grammar is installed for the given file's language. */
export declare function isGrammarSupported(filePath: string): boolean;
-export declare function parseFile(filePath: string, source: string): FileParseResult | null;
+export declare function parseFile(filePath: string, source: string, opts?: {
+ emitChunks?: boolean;
+}): FileParseResult | null;
export interface ResolvedCallEdge {
callerFilePath: string;
callerName: string;
@@ -85,8 +87,7 @@ export interface ResolvedEdge {
* - Unambiguous class method: dstQualifiedKey === 'ClassName.method'
* - Ambiguous (two entities share the same plain name): edge not emitted
*/
-/** Stats collected during resolution (exposed for debug instrumentation). */
-export interface ResolveStats {
+export declare function resolveEdges(results: FileParseResult[], stats?: {
importLookups: number;
transitiveLookups: number;
globalFallbacks: number;
@@ -97,8 +98,6 @@ export interface ResolveStats {
resolvedQualifier: number;
skippedSameFile: number;
skippedAmbiguous: number;
-}
-export declare function resolveEdges(results: FileParseResult[]): ResolvedEdge[];
-export declare function resolveEdges(results: FileParseResult[], statsOut: ResolveStats): ResolvedEdge[];
+}): ResolvedEdge[];
/** @deprecated Use resolveEdges instead. */
export declare function resolveCallEdges(results: FileParseResult[]): ResolvedCallEdge[];
diff --git a/core-ingestion/dist/index.js b/core-ingestion/dist/index.js
index 33d01b76..dabe5504 100644
--- a/core-ingestion/dist/index.js
+++ b/core-ingestion/dist/index.js
@@ -228,7 +228,7 @@ function unwrapRustCfgMacros(source) {
// ---------------------------------------------------------------------------
// Main parse function
// ---------------------------------------------------------------------------
-export function parseFile(filePath, source) {
+export function parseFile(filePath, source, opts) {
const language = languageFromPath(filePath);
if (!language)
return null;
@@ -548,27 +548,29 @@ export function parseFile(filePath, source) {
continue;
}
}
- for (const pendingChunk of pendingChunks) {
- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
- chunks.push({
- ...pendingChunk,
- contentHash,
- });
- }
- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
- if (chunks.length === 0) {
- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
- chunks.push({
- name: null,
- chunkKind: 'file_body',
- lineStart: 1,
- lineEnd: sourceLineCount,
- startByte: 0,
- endByte: source.length,
- contentHash,
- language,
- });
+ if (opts?.emitChunks !== false) {
+ for (const pendingChunk of pendingChunks) {
+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
+ chunks.push({
+ ...pendingChunk,
+ contentHash,
+ });
+ }
+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
+ if (chunks.length === 0) {
+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: sourceLineCount,
+ startByte: 0,
+ endByte: source.length,
+ contentHash,
+ language,
+ });
+ }
}
return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
}
diff --git a/core-ingestion/dist/patch-builder.d.ts b/core-ingestion/dist/patch-builder.d.ts
index 56c3ae9b..f7511c4b 100644
--- a/core-ingestion/dist/patch-builder.d.ts
+++ b/core-ingestion/dist/patch-builder.d.ts
@@ -4,4 +4,6 @@ export declare function extractorName(): string;
/** Previous extractor versions — their patches are superseded when re-ingesting. */
export declare const PREVIOUS_EXTRACTORS: string[];
export declare function buildPatch(result: FileParseResult, sourceHash: string, previousSourceHash?: string): GraphPatchPayload;
-export declare function buildPatchWithResolution(result: FileParseResult, sourceHash: string, resolvedEdges: ResolvedEdge[], previousSourceHash?: string): GraphPatchPayload;
+export declare function buildPatchWithResolution(result: FileParseResult, sourceHash: string, resolvedEdges: ResolvedEdge[], previousSourceHash?: string, opts?: {
+ emitClaims?: boolean;
+}): GraphPatchPayload;
diff --git a/core-ingestion/dist/patch-builder.js b/core-ingestion/dist/patch-builder.js
index 026cba0f..0fc438dd 100644
--- a/core-ingestion/dist/patch-builder.js
+++ b/core-ingestion/dist/patch-builder.js
@@ -239,7 +239,7 @@ export function buildPatch(result, sourceHash, previousSourceHash) {
// buildPatchWithResolution — like buildPatch but fixes CALLS edge dst to point
// to the actual defining file for cross-file calls resolved by resolveCallEdges.
// ---------------------------------------------------------------------------
-export function buildPatchWithResolution(result, sourceHash, resolvedEdges, previousSourceHash) {
+export function buildPatchWithResolution(result, sourceHash, resolvedEdges, previousSourceHash, opts) {
// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }
// Callers should pass only edges for this file (pre-grouped) for best performance,
// but we still tolerate the full array for backward compatibility.
@@ -381,15 +381,17 @@ export function buildPatchWithResolution(result, sourceHash, resolvedEdges, prev
attrs: {},
});
}
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
+ if (opts?.emitClaims !== false) {
+ for (const r of relationships) {
+ const srcKey = resolveKey(r.srcName);
+ ops.push({
+ type: 'AssertClaim',
+ entityId: nodeId(filePath, srcKey),
+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,
+ value: r.dstName,
+ confidence: null,
+ });
+ }
}
const extractor = extractorName();
const patchId = computePatchId(filePath, sourceHash, extractor);
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index dd12532c..78f05b10 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -283,7 +283,7 @@ function unwrapRustCfgMacros(source: string): string {
// Main parse function
// ---------------------------------------------------------------------------
-export function parseFile(filePath: string, source: string): FileParseResult | null {
+export function parseFile(filePath: string, source: string, opts?: { emitChunks?: boolean }): FileParseResult | null {
const language = languageFromPath(filePath);
if (!language) return null;
@@ -629,28 +629,30 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
}
}
- for (const pendingChunk of pendingChunks) {
- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
- chunks.push({
- ...pendingChunk,
- contentHash,
- });
- }
+ if (opts?.emitChunks !== false) {
+ for (const pendingChunk of pendingChunks) {
+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
+ chunks.push({
+ ...pendingChunk,
+ contentHash,
+ });
+ }
- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
- if (chunks.length === 0) {
- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
- chunks.push({
- name: null,
- chunkKind: 'file_body',
- lineStart: 1,
- lineEnd: sourceLineCount,
- startByte: 0,
- endByte: source.length,
- contentHash,
- language,
- });
+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
+ if (chunks.length === 0) {
+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: sourceLineCount,
+ startByte: 0,
+ endByte: source.length,
+ contentHash,
+ language,
+ });
+ }
}
return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.ts
index df704395..5500cb24 100644
--- a/core-ingestion/src/patch-builder.ts
+++ b/core-ingestion/src/patch-builder.ts
@@ -272,6 +272,7 @@ export function buildPatchWithResolution(
sourceHash: string,
resolvedEdges: ResolvedEdge[],
previousSourceHash?: string,
+ opts?: { emitClaims?: boolean },
): GraphPatchPayload {
// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }
// Callers should pass only edges for this file (pre-grouped) for best performance,
@@ -420,15 +421,17 @@ export function buildPatchWithResolution(
});
}
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
+ if (opts?.emitClaims !== false) {
+ for (const r of relationships) {
+ const srcKey = resolveKey(r.srcName);
+ ops.push({
+ type: 'AssertClaim',
+ entityId: nodeId(filePath, srcKey),
+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,
+ value: r.dstName,
+ confidence: null,
+ });
+ }
}
const extractor = extractorName();
diff --git a/ix-cli/src/cli/commands/explain.ts b/ix-cli/src/cli/commands/explain.ts
index f2663338..af5fbc0d 100644
--- a/ix-cli/src/cli/commands/explain.ts
+++ b/ix-cli/src/cli/commands/explain.ts
@@ -52,7 +52,7 @@ export function registerExplainCommand(program: Command): void {
console.log(JSON.stringify(output, null, 2));
} else {
if (facts.stale) {
- renderWarning("Source has changed since last ingest. Run ix ingest to update.");
+ renderWarning("Source has changed since last ingest. Run ix map to update.");
}
renderSection("Explanation");
@@ -213,7 +213,7 @@ async function rawExplain(
}
console.log(JSON.stringify(output, null, 2));
} else {
- if (stale) renderWarning("Source has changed since last ingest. Run ix ingest to update.");
+ if (stale) renderWarning("Source has changed since last ingest. Run ix map to update.");
formatExplain(result, "text");
}
}
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index d62f208f..3f809146 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -200,8 +200,20 @@ function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {
};
}
-function isWriteConflict(err: unknown): boolean {
- return String(err).includes('write-write conflict');
+const COMMIT_CONFLICT_RETRY_PATTERNS = [
+ 'write-write conflict',
+ 'timeout waiting to lock key',
+ 'error: 1200',
+ // Transport-level failures (k8s ingress reset, socket drop under load).
+ // Safe to retry because the server never received / committed the payload.
+ 'fetch failed',
+ 'econnreset',
+ 'econnrefused',
+];
+
+export function isRetryableCommitConflict(err: unknown): boolean {
+ const message = String(err).toLowerCase();
+ return COMMIT_CONFLICT_RETRY_PATTERNS.some(pattern => message.includes(pattern));
}
async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Promise<T> {
@@ -209,9 +221,9 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Pro
try {
return await fn();
} catch (err) {
- if (attempt >= maxRetries || !isWriteConflict(err)) throw err;
- // ArangoDB write-write conflicts resolve as soon as the competing transaction
- // commits — typically <10ms. Use short jittered delay to avoid thundering herd.
+ if (attempt >= maxRetries || !isRetryableCommitConflict(err)) throw err;
+ // ArangoDB lock conflicts usually clear as soon as the competing transaction
+ // commits. Transport errors may need a bit more breathing room under k8s load.
const delay = 10 * (1 << attempt) + Math.random() * 10;
await new Promise<void>(r => setTimeout(r, delay));
}
@@ -339,13 +351,6 @@ export async function ingestFiles(
let progressStart = performance.now();
const interval = opts.format === 'text' ? setInterval(() => {
- const elapsed = performance.now() - progressStart;
- const tau = progressPhase === 'Saving' ? 3000 : 2000;
- let display = progressCurrent;
- if (progressTotal > 0 && progressCurrent < progressTotal) {
- const simulated = Math.floor((1 - Math.exp(-elapsed / tau)) * 0.88 * progressTotal);
- display = Math.max(display, simulated);
- }
if (debug && currentWorkLabel) {
const workElapsed = performance.now() - currentWorkStart;
if (workElapsed >= SLOW_WORK_LOG_MS && (lastSlowWorkNotice === 0 || workElapsed - lastSlowWorkNotice >= SLOW_WORK_REPEAT_MS)) {
@@ -353,7 +358,7 @@ export async function ingestFiles(
process.stderr.write(`\n [slow ${progressPhase.toLowerCase()}] ${currentWorkLabel} (${Math.round(workElapsed / 1000)}s)\n`);
}
}
- process.stderr.write('\r' + renderProgressLine(progressPhase, display, progressTotal));
+ process.stderr.write('\r' + renderProgressLine(progressPhase, progressCurrent, progressTotal));
}, 80) : null;
let filesDiscovered = 0;
@@ -376,21 +381,41 @@ export async function ingestFiles(
};
// Phase timing marks
- const timings = { moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0, resolveMs: 0, buildPatchMs: 0, commitMs: 0 };
+ const timings = {
+ moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,
+ parseOnlyMs: 0, // sum of just parseFile() call durations
+ resolveMs: 0, buildPatchMs: 0, commitMs: 0,
+ stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)
+ bulkCommitMs: 0, // time in successful commitPatchBulk() calls
+ fallbackCommitMs: 0, // time in per-file fallback commitPatch() calls
+ };
const resolveStats = {
importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,
globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,
resolvedGlobal: 0, resolvedQualifier: 0, skippedSameFile: 0, skippedAmbiguous: 0,
};
+ // Per-language parse timing: language → { ms, files }
+ const langParseMs = new Map<string, { ms: number; files: number }>();
+ // Total graph op counts across all committed batches
+ let totalNodesEmitted = 0;
+ let totalEdgesEmitted = 0;
+ let totalClaimsEmitted = 0;
+
const logSaveTiming = (
files: number,
resolveEdgesMs: number,
buildPatchMs: number,
+ stripMs: number,
commitMs: number,
totalSaveMs: number,
+ nodes: number,
+ edges: number,
+ claims: number,
): void => {
process.stderr.write(
- `\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs} commitMs=${commitMs} totalSaveMs=${totalSaveMs}\n`
+ `\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` +
+ (mapMode ? ` stripChunkOpsMs=${stripMs}` : '') +
+ ` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`
);
};
@@ -492,7 +517,8 @@ export async function ingestFiles(
const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend
- const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 4); // parallel HTTP save requests
+ const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', mapMode ? 2 : 4); // parallel HTTP save requests
+ const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
const YIELD_EVERY = 100; // yield event loop every N files during parse
if (debug) {
@@ -532,6 +558,9 @@ export async function ingestFiles(
const commitMsPerChunk = new Array<number>(chunks.length).fill(0);
let nextChunk = 0;
+ // Fallback mutex: serializes per-file commits across all workers so that
+ // concurrent fallback loops don't race on revisions.current (Error 1200).
+ let fallbackTail: Promise<void> = Promise.resolve();
const runChunk = async (ci: number): Promise<void> => {
const chunk = chunks[ci];
@@ -543,8 +572,10 @@ export async function ingestFiles(
try {
setCurrentWork(`commit ${startFile}-${endFile} of ${totalFiles} ending ${endingPath}`);
const commitStart = performance.now();
- const result = await retryOnConflict(() => client.commitPatchBulk(patches), 3);
- commitMsPerChunk[ci] = Math.round(performance.now() - commitStart);
+ const result = await retryOnConflict(() => client.commitPatchBulk(patches), COMMIT_CONFLICT_RETRIES);
+ const chunkMs = Math.round(performance.now() - commitStart);
+ commitMsPerChunk[ci] = chunkMs;
+ timings.bulkCommitMs += chunkMs;
if (result.rev > latestRev) latestRev = result.rev;
patchesApplied += patches.length;
} catch (err) {
@@ -553,17 +584,30 @@ export async function ingestFiles(
`\n [bulk failed, falling back to per-file] files ${startFile}-${endFile} (${patches.length} patches): ${err}\n`
);
}
- for (const item of chunk) {
- try {
- const commitStart = performance.now();
- const result = await retryOnConflict(() => client.commitPatch(item.patch), 3);
- commitMsPerChunk[ci] += Math.round(performance.now() - commitStart);
- if (result.rev > latestRev) latestRev = result.rev;
- patchesApplied++;
- } catch (commitErr) {
- parseErrors++;
- process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${commitErr}\n`);
+ // Chain this chunk's fallback work onto the shared tail so that only one
+ // fallback loop runs at a time. This prevents concurrent commitPatch
+ // transactions from racing on revisions.current (write-write conflict).
+ const prev = fallbackTail;
+ let resolveThis!: () => void;
+ fallbackTail = new Promise<void>(r => { resolveThis = r; });
+ await prev;
+ try {
+ for (const item of chunk) {
+ try {
+ const commitStart = performance.now();
+ const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);
+ const chunkMs = Math.round(performance.now() - commitStart);
+ commitMsPerChunk[ci] += chunkMs;
+ timings.fallbackCommitMs += chunkMs;
+ if (result.rev > latestRev) latestRev = result.rev;
+ patchesApplied++;
+ } catch (commitErr) {
+ parseErrors++;
+ process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${commitErr}\n`);
+ }
}
+ } finally {
+ resolveThis();
}
}
@@ -591,13 +635,15 @@ export async function ingestFiles(
const flushBatch = async (batch: ParsedFile[]): Promise<void> => {
if (batch.length === 0) return;
filesChanged += batch.length;
- const previousPhase = progressPhase;
const saveStart = performance.now();
let resolveEdgesMs = 0;
let buildPatchMs = 0;
+ let localStripMs = 0;
let commitMs = 0;
+ let batchNodes = 0;
+ let batchEdges = 0;
+ let batchClaims = 0;
progressPhase = 'Saving';
- progressStart = performance.now();
try {
setCurrentWork(`resolve+commit batch ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
@@ -617,8 +663,7 @@ export async function ingestFiles(
for (let j = 0; j < batch.length; j++) {
const { parsed: p, hash, previousHash } = batch[j];
try {
- const rawPatch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
- const patch = mapMode ? stripChunkOps(rawPatch) : rawPatch;
+ const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
@@ -626,15 +671,26 @@ export async function ingestFiles(
}
}
buildPatchMs += Math.round(performance.now() - buildStart);
+ for (const pp of preparedPatches) {
+ for (const op of pp.patch.ops) {
+ if (op.type === 'UpsertNode') batchNodes++;
+ else if (op.type === 'UpsertEdge') batchEdges++;
+ else if (op.type === 'AssertClaim') batchClaims++;
+ }
+ }
if (preparedPatches.length === 0) return;
commitMs += await commitPreparedPatches(preparedPatches, batch.length);
} finally {
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
timings.buildPatchMs += buildPatchMs;
+ timings.stripChunkOpsMs += localStripMs;
timings.commitMs += commitMs;
- logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs);
- progressPhase = previousPhase;
+ totalNodesEmitted += batchNodes;
+ totalEdgesEmitted += batchEdges;
+ totalClaimsEmitted += batchClaims;
+ logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
+ progressPhase = 'Parsing';
progressStart = performance.now();
}
};
@@ -647,11 +703,14 @@ export async function ingestFiles(
const saveStart = performance.now();
let resolveEdgesMs = 0;
let buildPatchMs = 0;
+ let localStripMs = 0;
let commitMs = 0;
+ let batchNodes = 0;
+ let batchEdges = 0;
+ let batchClaims = 0;
progressPhase = 'Saving';
progressTotal = allParsed.length;
progressCurrent = 0;
- progressStart = performance.now();
try {
setCurrentWork(`resolve ${allParsed.length} files`);
const resolveStart = performance.now();
@@ -669,8 +728,7 @@ export async function ingestFiles(
for (let j = 0; j < allParsed.length; j++) {
const { parsed: p, hash, previousHash } = allParsed[j];
try {
- const rawPatch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash);
- const patch = mapMode ? stripChunkOps(rawPatch) : rawPatch;
+ const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
@@ -678,6 +736,13 @@ export async function ingestFiles(
}
}
buildPatchMs += Math.round(performance.now() - buildStart);
+ for (const pp of preparedPatches) {
+ for (const op of pp.patch.ops) {
+ if (op.type === 'UpsertNode') batchNodes++;
+ else if (op.type === 'UpsertEdge') batchEdges++;
+ else if (op.type === 'AssertClaim') batchClaims++;
+ }
+ }
if (preparedPatches.length === 0) {
progressCurrent = allParsed.length;
return;
@@ -687,8 +752,12 @@ export async function ingestFiles(
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
timings.buildPatchMs += buildPatchMs;
+ timings.stripChunkOpsMs += localStripMs;
timings.commitMs += commitMs;
- logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, commitMs, totalSaveMs);
+ totalNodesEmitted += batchNodes;
+ totalEdgesEmitted += batchEdges;
+ totalClaimsEmitted += batchClaims;
+ logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
progressPhase = previousPhase;
progressStart = performance.now();
}
@@ -732,8 +801,14 @@ export async function ingestFiles(
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
continue;
}
- const parsed = (ingestion.parseFile as Function)(filePath, sourceText);
+ const p0 = performance.now();
+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
+ const pfMs = performance.now() - p0;
if (!parsed) { filesSkipped++; progressCurrent++; continue; }
+ timings.parseOnlyMs += pfMs;
+ const lang: string = (parsed as any).language ?? 'unknown';
+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
entitiesParsed += parsed.entities.length;
allParsed.push({ filePath, parsed, hash, previousHash });
progressCurrent++;
@@ -776,8 +851,14 @@ export async function ingestFiles(
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
continue;
}
- const parsed = (ingestion.parseFile as Function)(filePath, sourceText);
+ const p0 = performance.now();
+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
+ const pfMs = performance.now() - p0;
if (!parsed) { filesSkipped++; progressCurrent++; continue; }
+ timings.parseOnlyMs += pfMs;
+ const lang: string = (parsed as any).language ?? 'unknown';
+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
entitiesParsed += parsed.entities.length;
const entry: ParsedFile = { filePath, parsed, hash, previousHash: previousHash !== hash ? previousHash : undefined };
if (useParseAllFirst) { allParsedB.push(entry); } else { batch.push(entry); }
@@ -833,7 +914,15 @@ export async function ingestFiles(
latestRev,
skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge, minifiedLikely },
elapsedSeconds: parseFloat(elapsed),
- timings,
+ timings: {
+ ...timings,
+ parseOnlyMs: Math.round(timings.parseOnlyMs),
+ stripChunkOpsMs: Math.round(timings.stripChunkOpsMs),
+ },
+ emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },
+ langParseMs: Object.fromEntries(
+ [...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])
+ ),
resolveStats,
}, null, 2));
} else if (opts.printSummary !== false) {
@@ -855,13 +944,30 @@ export async function ingestFiles(
if (debug) {
console.log(chalk.dim(`\n Phase timings:`));
- console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
- console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
- console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
- console.log(chalk.dim(` parse: ${timings.parseMs}ms`));
- console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
- console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));
- console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
+ console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
+ console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
+ console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
+ console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));
+ console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));
+ console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
+ console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));
+ if (mapMode) {
+ console.log(chalk.dim(` stripChunk: ${Math.round(timings.stripChunkOpsMs)}ms`));
+ }
+ console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
+ console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));
+ console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));
+ console.log(chalk.dim(`\n Emitted ops:`));
+ console.log(chalk.dim(` nodes: ${totalNodesEmitted}`));
+ console.log(chalk.dim(` edges: ${totalEdgesEmitted}`));
+ console.log(chalk.dim(` claims: ${totalClaimsEmitted}`));
+ if (langParseMs.size > 0) {
+ console.log(chalk.dim(`\n Per-language parse times:`));
+ const sorted = [...langParseMs.entries()].sort((a, b) => b[1].ms - a[1].ms);
+ for (const [lang, stat] of sorted) {
+ console.log(chalk.dim(` ${lang.padEnd(12)} ${Math.round(stat.ms)}ms (${stat.files} files)`));
+ }
+ }
console.log(chalk.dim(`\n Resolve stats:`));
console.log(chalk.dim(` import lookups: ${resolveStats.importLookups}`));
console.log(chalk.dim(` transitive lookups: ${resolveStats.transitiveLookups}`));
diff --git a/ix-cli/src/cli/commands/locate.ts b/ix-cli/src/cli/commands/locate.ts
index 0842a8f4..11096451 100644
--- a/ix-cli/src/cli/commands/locate.ts
+++ b/ix-cli/src/cli/commands/locate.ts
@@ -219,7 +219,7 @@ function outputLocate(output: LocateOutput, symbol: string, format: string): voi
return;
}
- if (output.stale) renderWarning("Some results may be stale. Run ix ingest to update.");
+ if (output.stale) renderWarning("Some results may be stale. Run ix map to update.");
if (!output.resolvedTarget) {
stderr(`No graph entity found for "${symbol}".`);
diff --git a/ix-cli/src/cli/commands/read.ts b/ix-cli/src/cli/commands/read.ts
index 88c8b46a..d67f5864 100644
--- a/ix-cli/src/cli/commands/read.ts
+++ b/ix-cli/src/cli/commands/read.ts
@@ -60,7 +60,7 @@ function outputResult(result: ReadResult, format: string): void {
if (format === "json") {
console.log(JSON.stringify(result, null, 2));
} else {
- if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix ingest to update.\n"));
+ if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix map to update.\n"));
if (result.targetType === "symbol" || result.targetType === "filename-match") {
stderr(chalk.dim(` ${result.path}:${result.lineStart}-${result.lineEnd}\n`));
}
diff --git a/ix-cli/src/cli/commands/status.ts b/ix-cli/src/cli/commands/status.ts
index 5fd9490c..824419e2 100644
--- a/ix-cli/src/cli/commands/status.ts
+++ b/ix-cli/src/cli/commands/status.ts
@@ -51,7 +51,7 @@ export function registerStatusCommand(program: Command): void {
if (staleInfo.staleFiles > staleInfo.sampleChangedFiles.length) {
renderNote(`... and ${staleInfo.staleFiles - staleInfo.sampleChangedFiles.length} more`);
}
- renderNote("Run ix ingest to update.");
+ renderNote("Run ix map to update.");
} else {
renderSuccess("Graph is up to date.");
}
diff --git a/ix-cli/src/cli/explain/render.ts b/ix-cli/src/cli/explain/render.ts
index 3a9c2bed..52f6fd0c 100644
--- a/ix-cli/src/cli/explain/render.ts
+++ b/ix-cli/src/cli/explain/render.ts
@@ -524,7 +524,7 @@ function renderNotes(facts: EntityFacts): string[] {
for (const d of facts.diagnostics) {
switch (d.code) {
case "unresolved_call_target":
- notes.push("Some downstream calls could not be resolved to named entities. Run `ix ingest` to improve coverage.");
+ notes.push("Some downstream calls could not be resolved to named entities. Run `ix map` to improve coverage.");
break;
case "stale_source":
notes.push("Source file has changed since last ingest — results may be incomplete.");
diff --git a/ix-cli/src/cli/explain/subsystem.ts b/ix-cli/src/cli/explain/subsystem.ts
index 8fe19052..f7d9cd74 100644
--- a/ix-cli/src/cli/explain/subsystem.ts
+++ b/ix-cli/src/cli/explain/subsystem.ts
@@ -311,7 +311,7 @@ function densityNote(chunkDensity: number): string {
return " High chunk density indicates thorough code coverage in the knowledge graph.";
}
if (chunkDensity < 2.0) {
- return " Low chunk density - run 'ix ingest' to improve coverage.";
+ return " Low chunk density - run 'ix ingest' to enable ix read / ix explain on this subsystem.";
}
return "";
}
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
index a9f38cd4..ee377976 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
@@ -84,6 +84,7 @@ class ArangoClient private (db: ArangoDatabase) {
val opts = new DocumentCreateOptions()
.overwriteMode(OverwriteMode.valueOf(overwriteMode))
.waitForSync(false)
+ .silent(true)
val docs = new java.util.ArrayList[java.util.Map[String, AnyRef]](documents.size)
documents.foreach(docs.add)
val result = db.collection(collection).insertDocuments(docs, opts)
@@ -243,7 +244,7 @@ object ArangoClient {
.host(host, port)
.user(user)
.password(if (password.isEmpty) null else password)
- .maxConnections(8)
+ .maxConnections(16)
.build()
val existingDbs = arango.getDatabases.asScala.toSet
if (!existingDbs.contains(database)) {
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
index 4a31b3e3..b6756095 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
@@ -308,7 +308,8 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {
val nodeIdStr = entityId.value.toString
client.query(
"""FOR p IN patches
- | FILTER LENGTH(
+ | FILTER @nodeId IN (IS_ARRAY(p.data.entityIds) ? p.data.entityIds : [])
+ | OR LENGTH(
| FOR op IN (IS_ARRAY(p.data.ops) ? p.data.ops : [])
| FILTER op.id == @nodeId OR op.entityId == @nodeId
| RETURN 1
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 3c92a0c9..7c2e64ba 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -96,10 +96,15 @@ class BulkWriteApi(client: ArangoClient) {
else timedUnit(tombstoneExistingNodes(logicalIds, newRev))
.both(timedUnit(retireOldClaims(fileBatches, newRev)))
(tombstoneMs: Long, retireClaimsMs: Long) = tombstoneAndRetireMs
- insertNodesMs <- insertCollectionChunked("nodes", allNodes, MaxNodeDocs)(docs => client.bulkInsert("nodes", docs))
- insertEdgesMs <- insertCollectionChunked("edges", allEdges, MaxEdgeDocs)(docs => client.bulkInsertEdges("edges", docs))
- insertClaimsMs <- insertCollectionChunked("claims", allClaims, MaxClaimDocs)(docs => client.bulkInsert("claims", docs))
- insertPatchesMs <- insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs))
+ // Run all four collection inserts concurrently — they write to independent collections
+ // with no referential-integrity enforcement between them. Wall time drops from the
+ // sum of all four to the max of any one (typically nodes or edges).
+ allInsertMs <- insertCollectionChunked("nodes", allNodes, MaxNodeDocs)(docs => client.bulkInsert("nodes", docs))
+ .both(insertCollectionChunked("edges", allEdges, MaxEdgeDocs)(docs => client.bulkInsertEdges("edges", docs)))
+ .both(insertCollectionChunked("claims", allClaims, MaxClaimDocs)(docs => client.bulkInsert("claims", docs)))
+ .both(insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs)))
+ .map { case (((n, e), c), p) => (n, e, c, p) }
+ (insertNodesMs, insertEdgesMs, insertClaimsMs, insertPatchesMs) = allInsertMs
updateRevMs <- timedUnit(updateRevision(newRev))
totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs + updateRevMs
_ <- logChunkTiming(
@@ -372,11 +377,68 @@ case class FileBatch(
}
def patchDocument(rev: Long): java.util.Map[String, AnyRef] = {
+ val source = new java.util.HashMap[String, AnyRef]()
+ source.put("uri", patch.source.uri)
+ source.put("sourceHash", patch.source.sourceHash.orNull)
+ source.put("extractor", patch.source.extractor)
+ source.put("sourceType", patch.source.sourceType.asJson.asString.getOrElse(patch.source.sourceType.toString))
+
+ val replaces = new java.util.ArrayList[String](patch.replaces.size)
+ patch.replaces.foreach(id => replaces.add(id.value.toString))
+
+ val entityIds = new java.util.LinkedHashSet[String]()
+ var nodeOpCount = 0
+ var edgeOpCount = 0
+ var claimOpCount = 0
+ patch.ops.foreach {
+ case PatchOp.UpsertNode(id, _, _, _) =>
+ nodeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.DeleteNode(id) =>
+ nodeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.UpsertEdge(id, _, _, _, _) =>
+ edgeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.DeleteEdge(id) =>
+ edgeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.AssertClaim(entityId, _, _, _, _) =>
+ claimOpCount += 1
+ entityIds.add(entityId.value.toString)
+ case PatchOp.RetractClaim(claimId) =>
+ claimOpCount += 1
+ entityIds.add(claimId.value.toString)
+ }
+
+ val entityIdList = new java.util.ArrayList[String](entityIds.size)
+ val entityIdIter = entityIds.iterator()
+ while (entityIdIter.hasNext) {
+ entityIdList.add(entityIdIter.next())
+ }
+
+ val data = new java.util.HashMap[String, AnyRef]()
+ data.put("patchId", patch.patchId.value.toString)
+ data.put("actor", patch.actor)
+ data.put("timestamp", patch.timestamp.toString)
+ data.put("source", source)
+ data.put("baseRev", Long.box(patch.baseRev.value))
+ data.put("replaces", replaces)
+ data.put("intent", patch.intent.orNull)
+ data.put("entityIds", entityIdList)
+ data.put("opCount", Int.box(patch.ops.size))
+ data.put("nodeOpCount", Int.box(nodeOpCount))
+ data.put("edgeOpCount", Int.box(edgeOpCount))
+ data.put("claimOpCount", Int.box(claimOpCount))
+
val doc = new java.util.HashMap[String, AnyRef]()
doc.put("_key", patch.patchId.value.toString)
doc.put("patch_id", patch.patchId.value.toString)
doc.put("rev", Long.box(rev))
- doc.put("data", jsonToJava(patch.asJson))
+ // Bulk ingest only needs patch metadata plus touched-entity IDs for provenance
+ // and source-hash lookups. Storing full ops here duplicates the graph payload
+ // and significantly inflates Kubernetes save latency.
+ doc.put("data", data)
doc: java.util.Map[String, AnyRef]
}
}
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
index 35ec4613..234fa82e 100644
--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
@@ -221,6 +221,38 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD
}
}
+ it should "return provenance chain for entity after bulk commit" in {
+ clientResource.use { client =>
+ val writeApi = new ArangoGraphWriteApi(client)
+ val queryApi = new ArangoGraphQueryApi(client)
+ val bulkWriteApi = new BulkWriteApi(client)
+ val app = buildRoutes(client, writeApi, queryApi)
+ val nodeId = NodeId(UUID.randomUUID())
+ val patch = makePatch(ops = Vector(PatchOp.UpsertNode(nodeId, NodeKind.Function, "bulk_prov_func", Map.empty[String, Json])))
+ val provenance = new java.util.HashMap[String, AnyRef]()
+ provenance.put("source_uri", patch.source.uri)
+ provenance.put("source_hash", patch.source.sourceHash.orNull)
+ provenance.put("extractor", patch.source.extractor)
+ provenance.put("source_type", patch.source.sourceType.asJson.asString.getOrElse("code"))
+ provenance.put("observed_at", patch.timestamp.toString)
+
+ for {
+ _ <- client.ensureSchema()
+ _ <- cleanDatabase(client)
+ _ <- bulkWriteApi.commitBatchChunked(
+ Vector(FileBatch(patch.source.uri, patch.source.sourceHash, patch, provenance)),
+ baseRev = 0L
+ )
+ resp <- app.run(Request[IO](Method.POST, Uri.unsafeFromString(s"/v1/provenance/${nodeId.value}")))
+ body <- resp.as[Json]
+ } yield {
+ resp.status shouldBe Status.Ok
+ body.hcursor.downField("entityId").as[String] shouldBe Right(nodeId.value.toString)
+ body.hcursor.downField("chain").focus.flatMap(_.asArray).exists(_.nonEmpty) shouldBe true
+ }
+ }
+ }
+
it should "create a decision node via POST /v1/decide" in {
clientResource.use { client =>
val app = buildRoutes(client, new ArangoGraphWriteApi(client), new ArangoGraphQueryApi(client))