Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): incremental adds, per-file baseline, parse budget, lazy pool, one discovery (#754 #775 #759 #763 #778)
Ian Hock1f3148d5d7b9
diff --git a/core-ingestion/src/__tests__/parseFile.perf.test.ts b/core-ingestion/src/__tests__/parseFile.perf.test.tsnew file mode 100644index 00000000..f7798e4a--- /dev/null+++ b/core-ingestion/src/__tests__/parseFile.perf.test.ts@@ -0,0 +1,92 @@+// Copyright 2026 Ix Infrastructure Inc.++import { describe, expect, it } from 'vitest';++import { parseFile } from '../index.js';++/**+ * Parse time must grow with the file, not with its square.+ *+ * Every use of an imported JS/TS name is checked for shadowing, and each check+ * used to rescan every enclosing scope in full: a test file with thousands of+ * calls to one imported helper took minutes, and held a parse worker the whole+ * time. These bounds are loose on purpose: the linear version takes well+ * under a second here, the quadratic one minutes, so a bound of 10 s holds on+ * a slow or busy CI runner and still fails the quadratic version. Each test+ * gets its own timeout, because vitest's default 5 s is less than the bound.+ */++/** The wall-clock bound for one parse, and the per-test timeout around it. */+const BOUND_MS = 10_000;+const TEST_TIMEOUT_MS = 60_000;++/** `n` calls to an imported function, all in one function body. */+function importedCalls(n: number): string {+ const lines = [`import { foo } from './x';`, `export function big() {`];+ for (let i = 0; i < n; i++) lines.push(` foo(${i});`);+ lines.push('}');+ return lines.join('\n');+}++/** A vitest-shaped file: `n` top-level-ish `it` blocks inside one `describe`. */+function vitestShaped(n: number): string {+ const lines = [`import { describe, it, expect } from 'vitest';`, `import { g } from './g';`, `describe('suite', () => {`];+ for (let i = 0; i < n; i++) {+ lines.push(` it('case ${i}', () => {`, ` const r = g(${i});`, ` expect(r).toBe(${i});`, ` });`);+ }+ lines.push('});');+ return lines.join('\n');+}++function timed(source: string): { ms: number; parsed: ReturnType<typeof parseFile> } {+ const start = performance.now();+ const parsed = parseFile('a/b.test.ts', source, { budgetMs: 0 });+ return { ms: performance.now() - start, parsed };+}++describe('parseFile scales linearly with uses of an imported name', () => {+ it('parses a 4,000-call file in seconds, not minutes', () => {+ const { ms, parsed } = timed(importedCalls(4000));+ expect(parsed).not.toBeNull();+ expect(ms).toBeLessThan(BOUND_MS);+ }, TEST_TIMEOUT_MS);++ it('parses a 1,000-test vitest file in seconds, not minutes', () => {+ const { ms, parsed } = timed(vitestShaped(1000));+ expect(parsed).not.toBeNull();+ expect(ms).toBeLessThan(BOUND_MS);+ }, TEST_TIMEOUT_MS);++ it('roughly quadruples in time when the file quadruples', () => {+ timed(importedCalls(500)); // warm the parser and query cache+ // Best of three each, so one scheduling hiccup cannot fail the ratio. A+ // factor of 4 rather than 2 keeps linear (about 4) and quadratic (about 16)+ // far apart, so load on a shared runner cannot push one across the bound.+ const best = (n: number) => Math.min(...[0, 1, 2].map(() => timed(importedCalls(n)).ms));+ const small = best(1000);+ const large = best(4000);+ expect(large / small).toBeLessThan(8);+ }, TEST_TIMEOUT_MS);+});++describe('parseFile budget', () => {+ it('gives up on a file that runs past its budget and says why', () => {+ const failures: Array<{ reason: string }> = [];+ const parsed = parseFile('a/b.test.ts', vitestShaped(2000), {+ budgetMs: 1,+ onFailure: (failure) => failures.push(failure),+ });+ expect(parsed).toBeNull();+ expect(failures.map((f) => f.reason)).toEqual(['timeout']);+ });++ it('a budget of 0 is no budget', () => {+ expect(parseFile('a/b.test.ts', vitestShaped(200), { budgetMs: 0 })).not.toBeNull();+ });++ it('a file no grammar handles is not a failure', () => {+ const failures: unknown[] = [];+ expect(parseFile('notes.unknownext', 'hello', { onFailure: (f) => failures.push(f) })).toBeNull();+ expect(failures).toEqual([]);+ });+});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex 7ed4b5dc..d1527967 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -706,9 +706,12 @@ function jsTsDeclarationBinds(node: any, name: string, namespace: 'value' | 'typreturn false;}+// `namedChildren` once rather than `namedChild(i)` per index: each indexed+// call walks the children from the start, so the loop was quadratic in the+// scope's size before it did any work.function jsTsScopeDirectlyBinds(scope: any, name: string, namespace: 'value' | 'type'): boolean {- for (let i = 0; i < scope.namedChildCount; i++) {- if (jsTsDeclarationBinds(scope.namedChild(i), name, namespace)) return true;+ for (const child of scope.namedChildren) {+ if (jsTsDeclarationBinds(child, name, namespace)) return true;}return false;}@@ -717,19 +720,50 @@ function jsTsFunctionHasVarBinding(node: any, name: string, isRoot = true): boolif (!node) return false;if (!isRoot && JS_TS_FUNCTION_NODES.has(node.type)) return false;if (node.type === 'variable_declaration' && jsTsDeclarationBinds(node, name, 'value')) return true;- for (let i = 0; i < node.namedChildCount; i++) {- if (jsTsFunctionHasVarBinding(node.namedChild(i), name, false)) return true;+ for (const child of node.namedChildren) {+ if (jsTsFunctionHasVarBinding(child, name, false)) return true;}return false;}+/**+ * Answers of the two whole-scope scans behind `isJsTsImportShadowed`, for one+ * parse: node id -> `${kind}\0${namespace}\0${name}` -> result. Without it each+ * USE of an imported name rescanned every enclosing scope, so a file with 4,000+ * `it(...)` calls scanned its 4,000 top-level statements 4,000 times. Node ids+ * are unique only within one tree, hence one memo per parse.+ */+type JsTsShadowMemo = Map<number, Map<string, boolean>>;++function memoisedScan(memo: JsTsShadowMemo | undefined, node: any, key: string, scan: () => boolean): boolean {+ if (!memo) return scan();+ let forNode = memo.get(node.id);+ if (!forNode) {+ forNode = new Map();+ memo.set(node.id, forNode);+ }+ let result = forNode.get(key);+ if (result === undefined) {+ result = scan();+ forNode.set(key, result);+ }+ return result;+}+/** True when a declaration shadows an imported JS/TS name at this use. */-function isJsTsImportShadowed(identifierNode: any, name: string, namespace: 'value' | 'type'): boolean {+function isJsTsImportShadowed(+ identifierNode: any,+ name: string,+ namespace: 'value' | 'type',+ memo?: JsTsShadowMemo,+): boolean {let child = identifierNode;let scope = identifierNode?.parent;while (scope) {if (scope.type === 'statement_block' || scope.type === 'program') {- if (jsTsScopeDirectlyBinds(scope, name, namespace)) return true;+ if (memoisedScan(memo, scope, `scope\0${namespace}\0${name}`, () => jsTsScopeDirectlyBinds(scope, name, namespace))) {+ return true;+ }}if (namespace === 'value' && scope.type === 'catch_clause') {if (jsTsBindingPatternContains(scope.childForFieldName?.('parameter'), name)) return true;@@ -743,7 +777,7 @@ function isJsTsImportShadowed(identifierNode: any, name: string, namespace: 'valif (scope.childForFieldName?.('name')?.text === name) return true;if (jsTsBindingPatternContains(scope.childForFieldName?.('parameters'), name)) return true;const body = scope.childForFieldName?.('body');- if (body && jsTsFunctionHasVarBinding(body, name)) return true;+ if (body && memoisedScan(memo, body, `var\0${name}`, () => jsTsFunctionHasVarBinding(body, name))) return true;} else if (jsTsTypeParametersContain(scope.childForFieldName?.('type_parameters'), name)) {return true;}@@ -2435,7 +2469,31 @@ function phpNamespaceForNode(node: any, spans: PhpNamespaceSpan[]): string | undreturn active;}-export function parseFile(filePath: string, source: string): FileParseResult | null {+/** Default per-file parse budget in ms, overridable with IX_PARSE_BUDGET_MS. */+export const DEFAULT_PARSE_BUDGET_MS = 10_000;++/** The per-file budget: IX_PARSE_BUDGET_MS if it is a non-negative number, else the default. 0 = none. */+export function parseBudgetMs(): number {+ const raw = process.env.IX_PARSE_BUDGET_MS;+ const n = raw === undefined || raw.trim() === '' ? Number.NaN : Number(raw);+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_PARSE_BUDGET_MS;+}++/** Thrown inside `parseFile` when a file runs past its budget; never escapes it. */+class ParseBudgetExceeded extends Error {}++export interface ParseFileOptions {+ /** Give up on the file after this many ms. Defaults to `parseBudgetMs()`; 0 turns the budget off. */+ budgetMs?: number;+ /**+ * Called when a supported file does not parse, before `parseFile` returns+ * null: `timeout` when it ran past the budget, `error` for anything else.+ * Not called for a file no grammar handles.+ */+ onFailure?: (failure: { reason: 'timeout' | 'error'; message: string }) => void;+}++export function parseFile(filePath: string, source: string, opts: ParseFileOptions = {}): FileParseResult | null {const language = detectLanguageForSource(filePath, source);if (!language) return null;if (language === SupportedLanguages.YAML) return parseYamlFile(filePath, source);@@ -2455,6 +2513,21 @@ export function parseFile(filePath: string, source: string): FileParseResult | nconst queries = LANGUAGE_QUERIES[language];if (!queries) return null;+ const budgetMs = opts.budgetMs ?? parseBudgetMs();+ const deadline = budgetMs > 0 ? performance.now() + budgetMs : Number.POSITIVE_INFINITY;+ let budgetTicks = 0;+ /**+ * Called once per match in both passes. The tree-sitter parse and query run+ * in native code and cannot be interrupted, but the passes over the matches+ * are where a file can go quadratic, and a file that does must not hold a+ * parse worker for minutes. The worker pool has a backstop for the rest.+ */+ const checkBudget = (): void => {+ if ((++budgetTicks & 63) === 0 && performance.now() > deadline) {+ throw new ParseBudgetExceeded(`parse budget of ${budgetMs} ms exceeded`);+ }+ };+try {const parser = getParser();if (grammar !== _currentGrammar) {@@ -2570,6 +2643,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n}}}+ const jsTsShadowMemo: JsTsShadowMemo = new Map();const recordJsTsImportUse = (predicate: string,srcName: string,@@ -2580,7 +2654,10 @@ export function parseFile(filePath: string, source: string): FileParseResult | n): void => {if (!isJsTs || !jsTsImportedLocalNames.has(localName)) return;const key = `${predicate}\x00${srcName}\x00${dstName}`;- const unshadowed = !isJsTsImportShadowed(identifierNode, localName, namespace);+ // Every use is still evaluated: the edge is kept when ANY use is+ // unshadowed, so the check cannot move behind the de-dup. The memo makes+ // each use cost its scope depth rather than the size of its scopes.+ const unshadowed = !isJsTsImportShadowed(identifierNode, localName, namespace, jsTsShadowMemo);jsTsImportUseHasUnshadowed.set(key, (jsTsImportUseHasUnshadowed.get(key) ?? false) || unshadowed);};@@ -2604,6 +2681,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n// --- First pass: collect definitions ---for (const match of pass1Matches) {+ checkBudget();// Definition captures: name + definition.*const defCapture = match.captures.find((c: any) =>c.name.startsWith('definition.')@@ -2879,6 +2957,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n// --- Second pass: calls and imports ---for (const match of pass2Matches) {+ checkBudget();// Full import statement captures (Scala: reconstructs dotted package paths)const importStmt = match.captures.find((c: any) => c.name === 'import.stmt');if (importStmt) {@@ -3381,6 +3460,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n};} catch (e) {if (process.env.IX_PARSE_DEBUG === '1') console.error('parseFile threw:', e);+ opts.onFailure?.({ reason: e instanceof ParseBudgetExceeded ? 'timeout' : 'error', message: String(e) });return null;}}diff --git a/core-ingestion/src/parse-worker.ts b/core-ingestion/src/parse-worker.tsindex bacbdbfb..7a0b925d 100644--- a/core-ingestion/src/parse-worker.ts+++ b/core-ingestion/src/parse-worker.ts@@ -5,7 +5,9 @@* Each worker maintains its own Parser singleton (safe — module state is* per-thread).* Receives: { filePath: string, source: string } | { __shutdown: true }- * Posts: { ok: true, result: FileParseResult } | { ok: false }+ * Posts: { ok: true, result: FileParseResult }+ * | { ok: false, result: null, reason?: 'timeout' | 'error', message?: string }+ * (no reason: no grammar handles the file)*/import { parentPort } from 'node:worker_threads';import { parseFile } from './index.js';@@ -88,10 +90,13 @@ parentPort.on('message', (msg: ParseMessage | ShutdownMessage) => {return;}const { filePath, source } = msg as ParseMessage;+ // Why a file came back null, so the pool can name the ones that ran past+ // the parse budget instead of folding them into "unparsed".+ let failure: { reason: 'timeout' | 'error'; message: string } | undefined;try {- const result = parseFile(filePath, source);- parentPort!.postMessage({ ok: result !== null, result: result ?? null });- } catch {- parentPort!.postMessage({ ok: false, result: null });+ const result = parseFile(filePath, source, { onFailure: (f) => { failure = f; } });+ parentPort!.postMessage({ ok: result !== null, result: result ?? null, ...(failure ?? {}) });+ } catch (err) {+ parentPort!.postMessage({ ok: false, result: null, reason: 'error', message: String(err) });}});diff --git a/docs-site/src/content/docs/reference/commands.md b/docs-site/src/content/docs/reference/commands.mdindex 612e2913..b309b5ad 100644--- a/docs-site/src/content/docs/reference/commands.md+++ b/docs-site/src/content/docs/reference/commands.md@@ -74,6 +74,11 @@ workspace is left alone. A path outside every workspace and repository is ingest`.gitignore` syntax: `#` comments, `*`, `?`, `**`, a leading `/` to anchor to the root, a trailing `/` fordirectories, and bare names that match at any depth. There is no `!` negation.+`ix ingest <path>` and `ix map` take the same per-workspace lock. A run that finds it held does not run beside+the holder: it asks the holder for one more pass, prints a note (or `{"coalesced": true, ...}` with+`--format json`) and exits 0, or with `IX_MAP_COALESCE_EXIT_CODE` if that is set. The holder runs that pass+before it exits: `ix map` re-ingests the workspace, and so does `ix ingest` when the workspace has been mapped.+## Setup and health| Command | Does |diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.mdindex 0dde0ceb..e0f922a9 100644--- a/docs-site/src/content/docs/reference/configuration.md+++ b/docs-site/src/content/docs/reference/configuration.md@@ -23,6 +23,8 @@ ix config set format llm # set a value| `IX_MCP_SUBPROCESS=1` | Runs each MCP tool call in its own `ix` process || `IX_COMMIT_FAILURE_LIMIT=<n>` | Consecutive failed commits before `ix map` stops (default 5). `0` never stops || `IX_COMMIT_BASE_REV_RETRIES=<n>` | Re-sends of a commit after another writer on the same backend moved the graph revision under it (default 8) |+| `IX_PARSE_BUDGET_MS=<ms>` | How long `ix map` / `ix ingest` may spend parsing one file before skipping it and naming it in the summary (default 10000). A skipped file is tried again on every run, and `ix status` warns about it until it is in the graph. `0` turns the budget off |+| `IX_PARSE_WORKERS=<n>` | Most parse workers `ix map` / `ix ingest` starts (default 8, never more than the cores less one). Workers start as files need them, one per 50 files |### Bootstrap script variablesdiff --git a/docs/api/README.md b/docs/api/README.mdindex d429efbb..47d22838 100644--- a/docs/api/README.md+++ b/docs/api/README.md@@ -167,7 +167,14 @@ Ingest a path into the graph. Long-running: the client allows **30 minutes**.> under the same key: `unchanged` there counts only files skipped as *mtime- or> hash-unchanged*, `emptyFile` is a real count rather than a hardcoded `0`, and> there is an extra `unparsed` bucket for files the parse pool returned nothing-> for. That is the client's own summary of its own run and is not this response;+> for. A file whose parse ran past `IX_PARSE_BUDGET_MS` is counted in+> `parseTimeout` and named in a top-level `parseTimeouts` list. Two buckets+> come from discovery and are not part of `filesSkipped`: `skippedDirs` counts+> files left out under a directory that is never source (build output,+> dependencies, VCS metadata, tool caches) -- by name on a plain walk, and only+> for untracked files in a git work tree, where a tracked file is always kept+> -- and `unreadable` counts listed files that could not be stat'd.+> That is the client's own summary of its own run and is not this response;> see the stitch section below for why the narrower `unchanged` is load-bearing> there.diff --git a/ix-cli/src/cli/__tests__/file-discovery.test.ts b/ix-cli/src/cli/__tests__/file-discovery.test.tsnew file mode 100644index 00000000..496a6ada--- /dev/null+++ b/ix-cli/src/cli/__tests__/file-discovery.test.ts@@ -0,0 +1,123 @@+// Copyright 2026 Ix Infrastructure Inc.++import { execFileSync } from 'node:child_process';+import { cpSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, relative } from 'node:path';++import { afterEach, describe, expect, it } from 'vitest';++import { discoverSourceFiles, emptyDiscoveryCounts, tryGitLsFiles, walkFiles } from '../file-discovery.js';++const scratch: string[] = [];+afterEach(() => {+ for (const dir of scratch.splice(0)) rmSync(dir, { recursive: true, force: true });+});++function tree(files: Record<string, string>): string {+ const root = realpathSync.native(mkdtempSync(join(tmpdir(), 'ix-files-')));+ scratch.push(root);+ for (const [rel, text] of Object.entries(files)) {+ mkdirSync(dirname(join(root, rel)), { recursive: true });+ writeFileSync(join(root, rel), text, 'utf8');+ }+ return root;+}++const git = (root: string, ...args: string[]) => execFileSync('git', args, { cwd: root, stdio: 'ignore' });+const rel = (root: string, files: string[]) => files.map(f => relative(root, f).replace(/\\/g, '/')).sort();++describe('file discovery', () => {+ it('keeps non-ASCII file names in a git repository (they used to be quoted and dropped)', () => {+ const root = tree({ 'src/café.ts': 'export const a = 1;\n', 'src/日本語.ts': 'export const b = 2;\n', 'src/plain.ts': '' });+ git(root, 'init', '-q');+ git(root, 'add', '-A');+ expect(rel(root, tryGitLsFiles(root, true)!)).toEqual(['src/café.ts', 'src/plain.ts', 'src/日本語.ts']);+ });++ it('git and the walk return the same files for the same tree', () => {+ const files = {+ '.gitignore': 'generated/\nlocal.ts\n',+ 'src/app.ts': '',+ 'src/local.ts': '',+ 'src/generated/out.ts': '',+ 'tests/test_app.py': '',+ 'pkg/server/server.go': '',+ 'examples/demo.ts': '',+ '.github/workflows/ci.yml': 'on: push\n',+ 'node_modules/dep/index.js': '',+ 'dist/bundle.js': '',+ 'pkg/api/types.pb.go': '',+ };+ const gitRoot = tree(files);+ git(gitRoot, 'init', '-q');+ const walkRoot = tree(files); // no .git: the walk++ const viaGit = rel(gitRoot, discoverSourceFiles(gitRoot));+ const viaWalk = rel(walkRoot, discoverSourceFiles(walkRoot));++ expect(viaWalk).toEqual(viaGit);+ expect(viaWalk).toEqual(['.github/workflows/ci.yml', 'examples/demo.ts', 'pkg/server/server.go', 'src/app.ts', 'tests/test_app.py']);+ });++ it('honours a nested .gitignore on the walk', () => {+ const root = tree({ 'a/.gitignore': 'secret.ts\n', 'a/secret.ts': '', 'a/kept.ts': '', 'b/secret.ts': '' });+ expect(rel(root, [...walkFiles(root, true)])).toEqual(['a/kept.ts', 'b/secret.ts']);+ });++ it('counts what it skipped instead of dropping it silently', () => {+ const root = tree({ 'src/a.ts': '', 'src/gone.ts': '' });+ git(root, 'init', '-q');+ git(root, 'add', '-A');+ rmSync(join(root, 'src', 'gone.ts')); // tracked, no longer on disk+ writeFileSync(join(root, 'src', 'b.js'), '', 'utf8');+ mkdirSync(join(root, 'dist'));+ writeFileSync(join(root, 'dist', 'b.js'), '', 'utf8'); // untracked build output+ const counts = emptyDiscoveryCounts();+ expect(rel(root, discoverSourceFiles(root, { counts }))).toEqual(['src/a.ts', 'src/b.js']);+ expect(counts).toEqual({ skippedDirs: 1, unreadable: 1 });+ });++ it('keeps a tracked file whatever its directory is called; skips untracked ones under those names', () => {+ // A repository that commits `build/` or `out/` is saying it is source.+ // git's ignore rules decide what is generated in a work tree; the names+ // are only a fallback for what nothing told git to ignore.+ const files = {+ 'build/gen.ts': '', 'out/main.ts': '', 'target/T.java': '', 'obj/x.cs': '', 'coverage/report.js': '',+ 'src/app.ts': '',+ };+ const root = tree(files);+ git(root, 'init', '-q');+ git(root, 'add', '-A');+ writeFileSync(join(root, 'build', 'fresh.ts'), '', 'utf8'); // untracked, not ignored+ const counts = emptyDiscoveryCounts();+ expect(rel(root, discoverSourceFiles(root, { counts }))).toEqual([+ 'build/gen.ts', 'coverage/report.js', 'obj/x.cs', 'out/main.ts', 'src/app.ts', 'target/T.java',+ ]);+ expect(counts.skippedDirs).toBe(1);++ // Without git there is no record of what is tracked: the walk keeps+ // skipping those names.+ const plain = tree(files);+ expect(rel(plain, discoverSourceFiles(plain))).toEqual(['src/app.ts']);+ });++ it('stops a walk at walkLimit, and never caps a git listing', () => {+ const root = tree({ 'a.ts': '', 'b.ts': '', 'c.ts': '', 'd.ts': '' });+ expect(discoverSourceFiles(root, { walkLimit: 2 })).toHaveLength(2);+ git(root, 'init', '-q');+ expect(discoverSourceFiles(root, { walkLimit: 2 })).toHaveLength(4);+ });++ it('copies of one tree agree whether or not they are a git checkout', () => {+ // The same tree as a git repository with committed files and as a plain+ // copy: discovery must not depend on how it was obtained.+ const root = tree({ 'lib/x.ts': '', 'lib/tests/x.test.ts': '', 'scripts/tool.py': '' });+ git(root, 'init', '-q');+ git(root, 'add', '-A');+ const copy = realpathSync.native(mkdtempSync(join(tmpdir(), 'ix-files-copy-')));+ scratch.push(copy);+ cpSync(root, copy, { recursive: true, filter: (src) => !src.includes(`${'/'}.git`) });+ expect(rel(copy, discoverSourceFiles(copy))).toEqual(rel(root, discoverSourceFiles(root)));+ });+});diff --git a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.tsindex 7dedc8e4..de47d427 100644--- a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts@@ -165,3 +165,16 @@ describe("extractor re-ingest progress", () => {expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);});});++describe("ingest baseline atomic write", () => {+ it("leaves no temp file behind when the rename fails", () => {+ const root = path.join(home, "p");+ const target = ingestMtimeCachePath(root);+ // A directory where the baseline goes makes the rename fail.+ fs.mkdirSync(target, { recursive: true });++ saveIngestBaseline(root, new Map([["a.ts", 1_000]]), 3);++ expect(fs.readdirSync(path.dirname(target)).filter(name => name.endsWith(".tmp"))).toEqual([]);+ });+});diff --git a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts b/ix-cli/src/cli/__tests__/ingest-equivalence.test.tsindex 9c53eaef..3b207b3e 100644--- a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts@@ -241,9 +241,9 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {await runSequence("head", [edit("editBody", "pkg/app.py"), edit("editBody", "src/com/ex/App.java")]);});- // F-07. A new file has no backend hash, so the whole repository takes the- // first-ingest path and every file is parsed and sent again.- it.fails("IN-01: adding a file sends one patch", async () => {+ // F-07. A new file has no backend hash; before IN-01 that sent the whole+ // repository down the first-ingest path, every file parsed and sent again.+ it("IN-01: adding a file sends one patch", async () => {await runSequence("head", [edit("addFile", undefined, 0)], {langs: ["ts"],check: (world) => {@@ -255,6 +255,14 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {});});+ // The index prescan holds no Python or Java symbols, so an added file in+ // either language still takes the whole-repository pass: resolved alone, its+ // cross-file calls would point at a module node or at nothing.+ it("IN-01: an added Python or Java file keeps its cross-file calls", async () => {+ await runSequence("head", [edit("addFile", undefined, 0)], { langs: ["py"] });+ await runSequence("head", [edit("addFile", undefined, 1)], { langs: ["java"] });+ });+// F-04. Dropping the file's last import removes a node, the patch carries a// DeleteNode, and a delete-bearing patch goes to `/v1/patch`, which does not// sweep: the removed CALLS and IMPORTS edges stay live.diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 28b6bd1a..831d103d 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -6,7 +6,8 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { execFileSync } from "node:child_process";-import { ingestFiles } from "../commands/ingest.js";+import { ingestFiles, ingestPathSingleFlight } from "../commands/ingest.js";+import { acquireMapLock, takeMapRerun } from "../single-flight.js";import { ingestMtimeCachePath, ingestRebuildPath, loadConfig } from "../config.js";import { workspaceIdForPath } from "../system.js";import { FakeBackend } from "./helpers/fake-backend.js";@@ -679,6 +680,121 @@ describe("ingestFiles against a fake backend", () => {expect(baselineFiles()).toEqual(expect.arrayContaining([join(repo, "src", "m000.ts"), join(repo, "src", "m002.ts")]));});+ it("names a file whose parse runs past the budget, ingests the rest, and retries it until it lands", async () => {+ // About 2.5 s of parsing, under the 1 MB cap, against a 1 s budget.+ // Whichever gives up first -- the worker's own check or the pool's+ // backstop at twice the budget --+ // the file must be named, not folded silently into "unparsed".+ fixture(3);+ const calls = Array.from({ length: 60_000 }, (_, i) => ` foo(${i});`);+ writeFileSync(join(repo, "src", "huge.ts"), [`import { foo } from './m000';`, "export function big() {", ...calls, "}"].join("\n"), "utf8");+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });+ const saved = process.env.IX_PARSE_BUDGET_MS;+ process.env.IX_PARSE_BUDGET_MS = "1000";+ const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ try {+ const summary = await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });++ expect(summary.parseTimeouts).toEqual(["src/huge.ts"]);+ expect(backend.acceptedPatches(), "the other three still land").toBe(3);+ expect(stderr.mock.calls.map(([c]) => String(c)).join("")).toContain("src/huge.ts");++ // Skipped, not settled: it used to be recorded clean, so every later run+ // skipped it as unchanged while status called the graph current.+ const huge = join(repo, "src", "huge.ts");+ const stored = loadIngestBaseline(repo)!;+ expect(stored.files.has(huge), "not recorded as ingested").toBe(false);+ expect(stored.parseTimeouts).toEqual(["src/huge.ts"]);+ expect(stored.pendingFiles).toEqual([]);+ const status = detectStaleFiles(repo);+ expect(status.parseTimeouts).toEqual(["src/huge.ts"]);+ expect(status.staleFiles, "named once, as a timeout").toBe(0);+ const llm = renderStatusLlm("ok", "x", status);+ expect(llm[0]).toContain("parse_timeouts=1");+ expect(llm[0]).toContain("stale=true");++ // With the budget off, the next run parses it and the warning goes.+ process.env.IX_PARSE_BUDGET_MS = "0";+ expect(backend.sourceUris).not.toContain("src/huge.ts");+ const next = await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ expect(next.parseTimeouts).toEqual([]);+ expect(backend.sourceUris, "retried, not skipped as unchanged").toContain("src/huge.ts");+ expect(loadIngestBaseline(repo)!.files.get(huge)).toBe(statSync(huge).mtimeMs);+ expect(loadIngestBaseline(repo)!.parseTimeouts).toEqual([]);+ expect(detectStaleFiles(repo).parseTimeouts).toEqual([]);+ } finally {+ stderr.mockRestore();+ if (saved === undefined) delete process.env.IX_PARSE_BUDGET_MS;+ else process.env.IX_PARSE_BUDGET_MS = saved;+ }+ });++ describe("a file the backend has never seen is an incremental change", () => {+ // A new file has no source hash on the backend, so `knownHashes` came back+ // empty and the whole repository took the first-ingest path: every file+ // re-read, re-parsed and re-sent to add one.+ const incremental = () =>+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const stage = () => execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });++ beforeEach(() => {+ fixture(16);+ backend.rememberHashes = true;+ });++ it("adding one file sends one patch", async () => {+ await incremental();+ writeFileSync(join(repo, "src", "added.ts"), "export const added = 1;\n", "utf8");+ stage();+ backend.resetRequests();++ const summary = await incremental();++ expect(backend.acceptedPatches(), "patches on the wire").toBe(1);+ expect(backend.commitCount).toBe(1);+ expect(summary.filesSkippedAsUnchanged).toBe(16);+ expect(summary.filesDiscovered).toBe(17);+ });++ it("an empty file that gains content sends one patch", async () => {+ writeFileSync(join(repo, "src", "empty.ts"), "", "utf8");+ stage();+ await incremental();+ writeFileSync(join(repo, "src", "empty.ts"), "export const filled = 1;\n", "utf8");+ backend.resetRequests();++ const summary = await incremental();++ expect(backend.acceptedPatches()).toBe(1);+ expect(summary.filesSkippedAsUnchanged).toBe(16);+ });++ it("a file that shrinks under the size cap sends one patch", async () => {+ writeFileSync(join(repo, "src", "big.ts"), `export const big = "${"x".repeat(1024 * 1024)}";\n`, "utf8");+ stage();+ await incremental();+ writeFileSync(join(repo, "src", "big.ts"), "export const big = 'small';\n", "utf8");+ backend.resetRequests();++ const summary = await incremental();++ expect(backend.acceptedPatches()).toBe(1);+ expect(summary.filesSkippedAsUnchanged).toBe(16);+ });++ it("a wiped backend still re-sends everything", async () => {+ await incremental();+ writeFileSync(join(repo, "src", "added.ts"), "export const added = 1;\n", "utf8");+ stage();+ backend.rememberHashes = false;+ backend.resetRequests();++ await incremental();++ expect(backend.acceptedPatches(), "the DB-reset guard empties the baseline").toBe(17);+ });+ });+describe("a changed file the backend answers Idempotent is reported, not marked clean (F-01)", () => {// A revert sends the patch id the backend committed for the original// bytes. The shipped backend replays any id it has seen, so it answers@@ -782,6 +898,190 @@ describe("ingestFiles against a fake backend", () => {});});+ describe("one failing file does not stop the baseline", () => {+ // The baseline used to be written only by a clean run. One unreadable file+ // then meant no run ever recorded anything again: every map re-read the+ // whole workspace, and deleted files were never retracted.+ const incremental = () =>+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const stage = () => execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });+ const stored = () => loadIngestBaseline(repo)!;+ let stderr: ReturnType<typeof vi.spyOn>;+ beforeEach(() => {+ stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ });+ afterEach(() => {+ stderr.mockRestore();+ process.exitCode = undefined;+ });++ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(+ "records every other file and retracts a deletion while one file is unreadable",+ async () => {+ fixture(3);+ // The graph model: a deletion reconciles through /v1/patches/:id and+ // /v1/entity/:id, which only it serves.+ backend.semantics = "head";+ const broken = join(repo, "src", "m000.ts");+ const deleted = join(repo, "src", "m001.ts");+ await incremental();+ const before = stored().files.get(broken);++ writeFileSync(broken, "export const changed = 1;\n", "utf8");+ rmSync(deleted);+ stage(); // before the chmod: git cannot read the file either+ execFileSync("chmod", ["000", broken]);+ backend.resetRequests();+ try {+ const summary = await incremental();+ expect(summary.parseErrors, "the unreadable file is an error").toBeGreaterThan(0);+ } finally {+ execFileSync("chmod", ["644", broken]);+ }++ const files = stored().files;+ expect(files.has(deleted), "the deletion was retracted and recorded").toBe(false);+ expect(backend.sourceUris).toContain("src/m001.ts");+ expect(files.get(join(repo, "src", "m002.ts")), "a settled file is recorded").toBeDefined();+ expect(files.get(broken), "the failed file keeps its old mtime").toBe(before);++ // Readable again: the next run sends it, and only it.+ backend.resetRequests();+ await incremental();+ expect(backend.acceptedPatches()).toBe(1);+ expect(stored().files.get(broken)).toBe(statSync(broken).mtimeMs);+ },+ );++ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(+ "a new file that fails is pending: retried, and ix status does not call the graph current",+ async () => {+ // A new file has no previous mtime to keep, so it dropped out of the+ // baseline while `lastIngestAt` moved past its mtime, and status+ // judged it current.+ fixture(3);+ await incremental();+ const added = join(repo, "src", "added.ts");+ writeFileSync(added, "export const added = 1;\n", "utf8");+ stage();+ execFileSync("chmod", ["000", added]);+ try {+ await incremental();+ } finally {+ execFileSync("chmod", ["644", added]);+ }++ expect(stored().files.has(added)).toBe(false);+ expect(stored().pendingFiles).toEqual(["src/added.ts"]);+ const status = detectStaleFiles(repo);+ expect(status.staleFiles).toBe(1);+ expect(status.sampleChangedFiles).toEqual(["src/added.ts"]);+ expect(renderStatusLlm("ok", "x", status)[0]).toContain("stale=true");++ // The next run sends it and clears the list.+ await incremental();+ expect(stored().files.get(added)).toBe(statSync(added).mtimeMs);+ expect(stored().pendingFiles).toEqual([]);+ expect(detectStaleFiles(repo).staleFiles).toBe(0);+ },+ );++ it("a failed lookup is not a reset", async () => {+ fixture(3);+ backend.rememberHashes = true;+ await incremental();+ backend.failSourceHashes = true;+ backend.resetRequests();++ const summary = await incremental();++ expect(summary.filesSkippedAsUnchanged, "still incremental").toBe(3);+ expect(backend.commitCount).toBe(0);+ });++ it("leaves no temp file beside the baseline", async () => {+ fixture(2);+ await incremental();+ const dir = join(home, ".ix");+ expect(existsSync(ingestMtimeCachePath(repo))).toBe(true);+ expect(execFileSync("ls", ["-a", dir], { encoding: "utf8" })).not.toMatch(/\.tmp/);+ });+ });++ describe("ix ingest coalesces with a map or ingest of the same workspace", () => {+ // It used to exit 1 with "run this again", which an editor hook or an MCP+ // client that ingests after every edit reported as a failed ingest, while+ // `ix map` in the same spot coalesces and exits 0.+ let stdout: ReturnType<typeof vi.spyOn>;+ let stderr: ReturnType<typeof vi.spyOn>;+ beforeEach(() => {+ stdout = vi.spyOn(console, "log").mockImplementation(() => undefined);+ stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ process.exitCode = undefined;+ });+ afterEach(() => {+ stdout.mockRestore();+ stderr.mockRestore();+ process.exitCode = undefined;+ delete process.env.IX_MAP_COALESCE_EXIT_CODE;+ });+ const written = (spy: ReturnType<typeof vi.spyOn>) =>+ spy.mock.calls.map((call: unknown[]) => String(call[0])).join("");++ it("exits 0, asks the holder for a rerun, and prints JSON a client can parse", async () => {+ fixture(1);+ const held = acquireMapLock(repo, "test");+ try {+ expect(held).not.toBeNull();+ backend.resetRequests();+ await ingestPathSingleFlight(join(repo, "src"), { format: "json" });++ expect(process.exitCode, "a coalesce is not a failure").toBeUndefined();+ expect(backend.commitCount, "nothing ran beside the holder").toBe(0);+ expect(JSON.parse(written(stdout))).toEqual({ coalesced: true, workspace: repo });+ expect(takeMapRerun(repo), "the holder is asked to run once more").toBe(true);+ } finally {+ held?.release();+ }+ });++ it("honours IX_MAP_COALESCE_EXIT_CODE, as ix map does", async () => {+ fixture(1);+ const held = acquireMapLock(repo, "test");+ try {+ process.env.IX_MAP_COALESCE_EXIT_CODE = "75";+ await ingestPathSingleFlight(repo, { format: "text" });+ expect(process.exitCode).toBe(75);+ expect(written(stderr)).toContain("Another ix map or ix ingest is running");+ } finally {+ held?.release();+ }+ });++ it("a holding ingest re-runs for the one that coalesced, so its edit lands", async () => {+ fixture(3);+ await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const first = join(repo, "src", "m000.ts");+ const second = join(repo, "src", "m002.ts");+ writeFileSync(first, "export const first = 1;\n", "utf8");+ writeFileSync(second, "export const second = 2;\n", "utf8");++ // The lock is taken before the first await, so the second call finds it+ // held whatever the scheduling.+ const holder = ingestPathSingleFlight(first, { format: "json" });+ const waiter = ingestPathSingleFlight(second, { format: "json" });+ await Promise.all([holder, waiter]);++ expect(process.exitCode).toBeUndefined();+ const files = loadIngestBaseline(repo)!.files;+ expect(files.get(second), "the coalesced run's file was ingested").toBe(statSync(second).mtimeMs);+ expect(files.get(first)).toBe(statSync(first).mtimeMs);+ const after = acquireMapLock(repo, "after");+ expect(after, "the lock was let go").not.toBeNull();+ after?.release();+ });+ });+it("ingests only the languages --lang names, and keeps the rest of the baseline", async () => {fixture(3);writeFileSync(join(repo, "src", "tool.py"), "def tool():\n return 1\n", "utf8");diff --git a/ix-cli/src/cli/__tests__/parse-pool-size.test.ts b/ix-cli/src/cli/__tests__/parse-pool-size.test.tsnew file mode 100644index 00000000..051ee9b9--- /dev/null+++ b/ix-cli/src/cli/__tests__/parse-pool-size.test.ts@@ -0,0 +1,32 @@+// Copyright 2026 Ix Infrastructure Inc.++import { describe, expect, it } from "vitest";++import { parsePoolSize } from "../commands/ingest.js";++describe("parsePoolSize", () => {+ it("gives a one-file edit one worker, however many cores there are", () => {+ expect(parsePoolSize(1, 24, undefined)).toBe(1);+ expect(parsePoolSize(50, 24, undefined)).toBe(1);+ expect(parsePoolSize(51, 24, undefined)).toBe(2);+ });++ it("caps a large run at 8 by default, or at IX_PARSE_WORKERS", () => {+ expect(parsePoolSize(10_000, 24, undefined)).toBe(8);+ expect(parsePoolSize(10_000, 24, "16")).toBe(16);+ expect(parsePoolSize(10_000, 24, "2")).toBe(2);+ });++ it("never exceeds the cores less one, and never drops below one", () => {+ expect(parsePoolSize(10_000, 4, "16")).toBe(3);+ expect(parsePoolSize(10_000, 2, undefined)).toBe(1);+ expect(parsePoolSize(10_000, 1, undefined)).toBe(1);+ expect(parsePoolSize(0, 24, undefined)).toBe(1);+ });++ it("ignores an IX_PARSE_WORKERS that is not a positive integer", () => {+ for (const bad of ["0", "-3", "1.5", "lots", ""]) {+ expect(parsePoolSize(10_000, 24, bad), bad).toBe(8);+ }+ });+});diff --git a/ix-cli/src/cli/__tests__/parse-pool.test.ts b/ix-cli/src/cli/__tests__/parse-pool.test.tsindex 1fcbb3dd..8d479e1c 100644--- a/ix-cli/src/cli/__tests__/parse-pool.test.ts+++ b/ix-cli/src/cli/__tests__/parse-pool.test.ts@@ -868,4 +868,109 @@ describe("ParsePool", () => {expect(elapsed).toBeLessThan(3000);expect(pool.crashedTasks(), "a clean teardown is not a crash").toBe(0);}, 20000);++ /**+ * Blocks its event loop for 1.5 s on `slow.ts` -- standing in for a parse+ * stuck in native code, which no budget check inside the worker can reach --+ * then answers. Every other file answers at once.+ */+ const SLOW_ON_ONE = `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ if (msg.filePath === 'slow.ts') Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `;++ it("settles a parse that holds its worker past the backstop as a timeout, and carries on", async () => {+ // The backstop also runs for `a.ts` from the moment it is handed to the+ // replacement worker, whose start-up counts against it. 1 s leaves room for+ // that start on a busy runner; the slow parse holds its worker for 6 s.+ const pool = new ParsePool(worker("slow", SLOW_ON_ONE), 1, undefined, undefined, 1000);+ pool.init();+ const start = Date.now();++ const [slow, next] = await Promise.all([pool.parse("slow.ts", "x"), pool.parse("a.ts", "y")]);++ expect(slow, "settled, not left pending").toBeNull();+ expect(next, "a replacement worker took the queue").toEqual({ filePath: "a.ts" });+ expect(Date.now() - start, "did not wait for the slow parse").toBeLessThan(5000);+ expect(pool.timedOutFiles()).toEqual(["slow.ts"]);+ expect(pool.crashedTasks(), "a timeout is not a crash: it recurs every run").toBe(0);+ await pool.destroy();+ });++ /** Reports a budget timeout for `t.ts`, as the real worker does. */+ const REPORTS_TIMEOUT = `+ import { parentPort } from 'node:worker_threads';+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ if (msg.filePath === 't.ts') {+ parentPort.postMessage({ ok: false, result: null, reason: 'timeout', message: 'budget' });+ } else {+ parentPort.postMessage({ ok: false, result: null, reason: 'error', message: 'boom' });+ }+ });+ `;++ /** Echoes, and appends a line to `marker` when it starts: one line per worker. */+ const COUNTS_STARTS = (marker: string) => `+ import { parentPort, threadId } from 'node:worker_threads';+ import { appendFileSync } from 'node:fs';+ appendFileSync(${JSON.stringify(marker)}, threadId + '\\n');+ parentPort.on('message', (msg) => {+ if (msg && msg.__shutdown) { parentPort.close(); return; }+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });+ });+ `;++ it("starts workers only as work arrives, up to its size", async () => {+ // A one-file edit used to start a worker per core to parse one file.+ const marker = join(dir, "starts");+ const pool = new ParsePool(worker("counts", COUNTS_STARTS(marker)), 4);+ const started = () => (existsSync(marker) ? readFileSync(marker, "utf8").trim().split("\n").length : 0);++ await pool.parse("a.ts", "x");+ expect(started(), "one task, one worker").toBe(1);++ await Promise.all(["b", "c", "d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));+ expect(started(), "never more than the pool's size").toBeLessThanOrEqual(4);++ await pool.destroy();+ });++ it("grows when a later phase asks for more, and never shrinks", async () => {+ const marker = join(dir, "grows");+ const pool = new ParsePool(worker("grows", COUNTS_STARTS(marker)), 1);+ const started = () => readFileSync(marker, "utf8").trim().split("\n").length;++ await Promise.all(["a", "b", "c"].map((f) => pool.parse(`${f}.ts`, "x")));+ expect(started()).toBe(1);++ pool.growTo(3);+ pool.growTo(2);+ await Promise.all(["d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));+ expect(started()).toBeGreaterThan(1);+ expect(started()).toBeLessThanOrEqual(3);++ await pool.destroy();+ });++ it("names a file the worker reports as timed out, and only one that counts", async () => {+ const pool = new ParsePool(worker("reports", REPORTS_TIMEOUT), 1);+ pool.init();++ const results = await Promise.all([+ pool.parse("t.ts", "x"),+ pool.parse("e.ts", "x"),+ // The index prescan does not count its losses; see `parse`.+ pool.parse("t.ts", "x", false),+ ]);++ expect(results).toEqual([null, null, null]);+ expect(pool.timedOutFiles()).toEqual(["t.ts"]);+ expect(pool.crashedTasks()).toBe(0);+ await pool.destroy();+ });});diff --git a/ix-cli/src/cli/__tests__/stale-workspace.test.ts b/ix-cli/src/cli/__tests__/stale-workspace.test.tsindex 0d931555..d7d97342 100644--- a/ix-cli/src/cli/__tests__/stale-workspace.test.ts+++ b/ix-cli/src/cli/__tests__/stale-workspace.test.ts@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";import * as fs from "node:fs";import * as os from "node:os";import * as path from "node:path";+import { execFileSync } from "node:child_process";import { clearIngestMtimeCache, clearMapBaseline, ingestMtimeCachePath, saveConfig } from "../config.js";import { loadIngestBaseline } from "../ingest-baseline.js";@@ -77,6 +78,7 @@ describe("workspace-scoped staleness", () => {staleFiles: 1,sampleChangedFiles: ["a.js"],replayedFiles: [],+ parseTimeouts: [],});expect(@@ -141,6 +143,7 @@ describe("workspace-scoped staleness", () => {staleFiles: 0,sampleChangedFiles: [],replayedFiles: [],+ parseTimeouts: [],});// The workspace is unverified, but no individual file is known to have// changed — the distinction this pair of assertions exists to pin down.@@ -203,6 +206,7 @@ describe("workspace-scoped staleness", () => {staleFiles: 0,sampleChangedFiles: [],replayedFiles: [],+ parseTimeouts: [],});});@@ -276,6 +280,7 @@ describe("workspace-scoped staleness", () => {staleFiles: 1,sampleChangedFiles: ["deleted.js"],replayedFiles: [],+ parseTimeouts: [],});// From inside the workspace, as a read runs. The probe resolves the active// root from cwd, and a repository cwd now outranks `default: true` (see@@ -317,6 +322,8 @@ describe("workspace-scoped staleness", () => {tracksMapBaseline: true,extractor: "tree-sitter/9.9",replayedFiles: [],+ pendingFiles: [],+ parseTimeouts: [],});});@@ -340,6 +347,67 @@ describe("workspace-scoped staleness", () => {expect(loadIngestBaseline(root)?.currentRev).toBe(20);});+ it("judges a root reached through a symlink by its own relative paths", () => {+ // macOS's tmpdir (/var -> /private/var) and Windows 8.3 names are roots+ // whose canonical path differs. Discovery returns canonical paths; the+ // baseline may hold either spelling, and the report must say `a.js`.+ const realRoot = path.join(home, "real");+ const linkedRoot = path.join(home, "linked");+ writeSource(realRoot, "a.js", "export const a = 1;\n");+ writeSource(realRoot, "b.js", "export const b = 1;\n");+ writeSource(realRoot, "gone.js", "export const gone = 1;\n");+ fs.symlinkSync(realRoot, linkedRoot, "junction");+ const fileA = path.join(linkedRoot, "a.js");+ const fileB = fs.realpathSync.native(path.join(realRoot, "b.js"));+ const gone = path.join(linkedRoot, "gone.js");++ expect(+ persistIngestBaselineIfClean(+ linkedRoot,+ new Map([+ [fileA, fs.statSync(fileA).mtimeMs],+ [fileB, fs.statSync(fileB).mtimeMs],+ [gone, fs.statSync(gone).mtimeMs],+ ]),+ 3,+ 0,+ 0,+ new Date(Date.now() + 60_000),+ ),+ ).toBe(true);+ expect(detectStaleFiles(linkedRoot).staleFiles).toBe(0);++ moveMtimeForward(fileA);+ fs.rmSync(gone);+ const result = detectStaleFiles(linkedRoot);+ expect(result.staleFiles).toBe(2);+ expect([...result.sampleChangedFiles].sort()).toEqual(["a.js", "gone.js"]);+ });++ it("stops a walk at its cap without calling the files past it deleted; a git listing is not capped", () => {+ // Status runs on every agent turn. Without a cap, a walk of a directory+ // that is not a work tree -- a home directory, say -- reads the whole disk.+ const root = path.join(home, "capped");+ const files = ["a.js", "b.js", "c.js", "d.js", "e.js"].map((name) => writeSource(root, name, ""));+ persistIngestBaselineIfClean(+ root,+ new Map(files.map((f) => [f, fs.statSync(f).mtimeMs])),+ 4,+ 0,+ 0,+ new Date(Date.now() + 60_000),+ );+ for (const f of files) moveMtimeForward(f);++ expect(detectStaleFiles(root, 5).staleFiles, "uncapped, every edit is seen").toBe(5);+ const capped = detectStaleFiles(root, 5, 3);+ expect(capped.staleFiles, "only the files read, and none reported deleted").toBe(3);++ execFileSync("git", ["init", "-q"], { cwd: root, stdio: "ignore" });+ execFileSync("git", ["add", "-A"], { cwd: root, stdio: "ignore" });+ expect(detectStaleFiles(root, 5, 3).staleFiles, "git lists the whole repository").toBe(5);+ });+it("uses ingest time for files absent from the mtime cache", async () => {const root = path.join(home, "partial-cache");const mappedFile = writeSource(root, "mapped.js", "export const mapped = true;\n");diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex d33b5d28..937d0af0 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -4,7 +4,6 @@ import * as nodePath from 'node:path';import * as fs from 'node:fs';import * as crypto from 'node:crypto';import * as os from 'node:os';-import { spawnSync } from 'node:child_process';import { fileURLToPath } from 'node:url';import type { Command } from 'commander';import { ParsePool } from './parse-pool.js';@@ -14,9 +13,14 @@ import { IxClient } from '../../client/api.js';import type { GraphPatchPayload } from '../../client/types.js';import { canonicalWorkspacePath, isPathInside, resolveWorkspaceRoot, clearMapResultCache, clearStitchScopeCache } from '../config.js';import { resolveIngestRoot } from '../map-root.js';+import { acquireMapLock, requestMapRerun, takeMapRerun } from '../single-flight.js';+import { applyRequestedMapCoalesceExitCode } from './map.js';+import {+ canonicalizeDiscoveredFilePath, discoverSourceFiles, emptyDiscoveryCounts, isSupportedSourceFile,+} from '../file-discovery.js';import {clearRebuildProgress, extractorChanged, isRev, loadIngestBaseline, loadRebuildProgress,- saveIngestBaseline, saveRebuildProgress,+ saveIngestBaseline, saveRebuildProgress, type BaselineFileNotes,} from '../ingest-baseline.js';import { resolveGitHubToken } from '../github/auth.js';import { parseGitHubRepo, fetchGitHubData } from '../github/fetch.js';@@ -35,10 +39,8 @@ import {perFileAction,} from '../commit-breaker.js';import { readBackendHealth } from './upgrade.js';-import { SUPPORTED_EXTENSIONS } from '../supported-extensions.js';import {- createIgnoreMatcher, parseIgnoreFile, parseIgnorePattern,- type IgnoreMatcher, type IgnorePattern,+ createIgnoreMatcher, parseIgnoreFile, parseIgnorePattern, type IgnorePattern,} from '../ignore-globs.js';import { canRenderProgress } from '../stderr.js';import { createTypeScriptModuleResolver } from '../ts-module-resolution.js';@@ -57,14 +59,7 @@ import { createClient } from "../../client/factory.js";// File discovery// ----------------------------------------------------------------------------export function isSupportedSourceFile(filePath: string): boolean {- const fileName = nodePath.basename(filePath).toLowerCase();- return fileName === 'dockerfile'- || fileName.endsWith('.dockerfile')- || fileName === 'makefile'- || fileName === 'gnumakefile'- || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());-}+export { isSupportedSourceFile, tryGitLsFiles, walkFiles } from '../file-discovery.js';// Extensions whose source text must be pre-read so buildGlobalResolutionIndex can// extract cross-batch symbols before the streaming parse loop. Go uses a fast@@ -107,64 +102,29 @@ function parseLangs(raw: string): Set<string> {);}-const IGNORE_DIRS = new Set([- 'node_modules', '.git', '.hg', '.svn', 'dist', 'build', 'target', 'out', '.next',- '__pycache__', '.tox', '.venv', 'venv', '.mypy_cache', '.pytest_cache',- '.gradle', '.idea', '.vscode', '.settings', '.vs',- '.cache', '.parcel-cache', 'coverage', '.nyc_output',- 'vendor', 'Pods', '.dart_tool', '.pub-cache',- 'bin', 'obj', 'pkg',- '.ix', '.claude', '.gitnexus',- 'test', 'tests', '__tests__', 'spec', 'specs', 'e2e',- 'examples', 'fixtures', '__mocks__', '__fixtures__',-]);--// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.-// These files inflate parse time and graph size without contributing to architectural signal.-const GENERATED_FILE_SUFFIXES = [- '.pb.go', // protobuf generated Go- '_deepcopy.go', // controller-gen deepcopy (Kubernetes ecosystem)- '_mock.go', // gomock generated mocks- '.pb.ts', // protobuf generated TypeScript- '.pb.js', // protobuf generated JavaScript- '_pb.ts', // alternative protobuf TypeScript naming- '_pb.js', // alternative protobuf JavaScript naming-];-const GENERATED_FILE_PREFIXES = [- 'zz_generated', // controller-gen convention (Kubernetes ecosystem)- 'mock_', // gomock convention-];+const MAX_FILE_BYTES = 1024 * 1024; // 1 MB-// Generated dependency lockfiles, matched whole. Only the ones whose extension-// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,-// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end-// in `.lock`, which is not a discovered extension, so they never reach here.-// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger-// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and-// it describes a resolved dependency snapshot, not architecture.-const GENERATED_FILE_NAMES = new Set([- 'package-lock.json', // npm- 'npm-shrinkwrap.json', // npm, published form- 'pnpm-lock.yaml', // pnpm- 'packages.lock.json', // NuGet-]);+/** Parse workers at most, unless IX_PARSE_WORKERS says otherwise. */+const DEFAULT_PARSE_WORKERS = 8;+/** Files a parse worker should have before another one is worth starting. */+const FILES_PER_PARSE_WORKER = 50;-function isGeneratedFile(basename: string): boolean {- // Lowercased for the name match alone: `isSupportedSourceFile` lowercases- // too, so on a case-insensitive filesystem a `Package-lock.json` is- // discovered and would otherwise slip past a case-sensitive comparison.- if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;- for (const suffix of GENERATED_FILE_SUFFIXES) {- if (basename.endsWith(suffix)) return true;- }- for (const prefix of GENERATED_FILE_PREFIXES) {- if (basename.startsWith(prefix)) return true;- }- return false;+/**+ * How many parse workers `expectedTasks` files deserve: one per 50 files,+ * never more than the cores less one or IX_PARSE_WORKERS (default 8), never+ * fewer than one. Each worker loads every grammar, about 70 MB, so a worker+ * per core for a one-file edit cost more than the parse.+ */+export function parsePoolSize(+ expectedTasks: number,+ cores: number = os.availableParallelism(),+ configured: string | undefined = process.env.IX_PARSE_WORKERS,+): number {+ const fromEnv = Number(configured);+ const cap = configured !== undefined && Number.isInteger(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_PARSE_WORKERS;+ return Math.max(1, Math.min(cores - 1, cap, Math.ceil(expectedTasks / FILES_PER_PARSE_WORKER)));}--const MAX_FILE_BYTES = 1024 * 1024; // 1 MBconst MINIFIED_BYTES_THRESHOLD = 200 * 1024;const MINIFIED_MAX_LINE_THRESHOLD = 20_000;const MINIFIED_AVG_LINE_THRESHOLD = 2_000;@@ -181,32 +141,6 @@ export function parsePositiveIntEnv(name: string, fallback: number): number {return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;}-export function* walkFiles(- dir: string,- recursive: boolean,- exclude?: { matcher: IgnoreMatcher; root: string; onSkip?: () => void },-): Generator<string> {- let entries: fs.Dirent[];- try { entries = fs.readdirSync(dir, { withFileTypes: true }); }- catch { return; }-- for (const entry of entries) {- if (entry.isDirectory() && entry.name.startsWith('.') && entry.name !== '.') continue;- if (IGNORE_DIRS.has(entry.name)) continue;- const full = nodePath.join(dir, entry.name);- // Directories are tested before descending, so an excluded tree costs one- // check rather than one per file inside it.- if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), entry.isDirectory())) {- exclude.onSkip?.();- continue;- }- if (entry.isDirectory()) {- if (recursive) yield* walkFiles(full, true, exclude);- } else if (entry.isFile()) {- if (isSupportedSourceFile(entry.name) && !isGeneratedFile(entry.name)) yield full;- }- }-}/*** The patterns in force for this ingest: `--exclude` flags first, then the@@ -229,13 +163,6 @@ export function collectExcludePatterns(root: string, cliPatterns: string[] = [])return patterns;}-function canonicalizeDiscoveredFilePath(filePath: string): string {- try {- return fs.realpathSync.native(filePath);- } catch {- return nodePath.resolve(filePath);- }-}/*** True when `candidate` lies inside `root`.@@ -307,67 +234,11 @@ export function discoverIngestFilePaths(// root drops everything whenever the root is itself reached through a link —// macOS `/var` -> `/private/var`, a home on a network mount, a `~/code`// symlink — which would look like "this repo has no source files".- const canonicalRoot = canonicalize(root);+ const canonicalRoot = canonicalizeDiscoveredFilePath(root);const files = canonical.filter((candidate) => isWithinDiscoveryRoot(canonicalRoot, candidate));return { files, outsideRoot: canonical.length - files.length };}-/**- * Exported for the discovery tests: the symlink-containment guarantee is a- * property of git-listing -> canonicalize -> confine *composed*, and each step- * looks fine alone. A test that cannot run the real listing cannot show that a- * committed symlink escapes in the first place.- */-export function tryGitLsFiles(- dir: string,- recursive: boolean,- exclude?: { matcher: IgnoreMatcher; root: string; onSkip?: () => void },-): string[] | null {- try {- const result = spawnSync(- 'git',- // -c: a cloned repository's own .git/config can name a core.fsmonitor- // command, which ls-files would run. Never let it.- ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', 'ls-files', '--cached', '--others', '--exclude-standard'],- {- cwd: dir,- encoding: 'utf-8',- maxBuffer: 32 * 1024 * 1024,- }- );- if (result.status !== 0) return null;-- const files: string[] = [];- const seenCanonical = new Set<string>();- for (const line of result.stdout.split(/\r?\n/)) {- if (!line) continue;- const fullPath = nodePath.resolve(dir, line);- if (!recursive && nodePath.dirname(fullPath) !== dir) continue;- // Before the extension check, so the count reports what the caller- // excluded rather than what they excluded and Ix would have skipped- // anyway. `git ls-files` never saw IGNORE_DIRS either, which is why a- // git repository's `tests/` tree reaches the graph today.- if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {- exclude.onSkip?.();- continue;- }- if (!isSupportedSourceFile(fullPath)) continue;- if (isGeneratedFile(nodePath.basename(fullPath))) continue;- try {- if (!fs.statSync(fullPath).isFile()) continue;- const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);- if (seenCanonical.has(canonicalPath)) continue;- seenCanonical.add(canonicalPath);- files.push(canonicalPath);- } catch {- // Ignore racy deletes and permission issues during discovery.- }- }- return files;- } catch {- return null;- }-}function sha256(content: Buffer): string {return crypto.createHash('sha256').update(content).digest('hex');@@ -971,9 +842,25 @@ export function persistIngestBaselineIfClean(now?: Date,deletedFiles: Map<string, string[]> = new Map(),extractor?: string | null,- replayedFiles: readonly string[] = [],+ notes: BaselineFileNotes = {},): boolean {if (!ingestCompletedCleanly(parseErrors, commitErrors)) return false;+ return persistIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);+}++/**+ * Write the baseline -- whatever errors the run had. The caller has already+ * left the files it did not settle at their previous mtimes.+ */+export function persistIngestBaseline(+ projectRoot: string,+ mtimes: Map<string, number>,+ currentRev: number,+ now?: Date,+ deletedFiles: Map<string, string[]> = new Map(),+ extractor?: string | null,+ notes: BaselineFileNotes = {},+): boolean {// An empty mtime map is normally a discovery failure, not an empty repo —// an over-broad ignore rule, the wrong cwd, a glob that matched nothing —// and persisting it silently discards the baseline, forcing a full re-ingest@@ -985,7 +872,7 @@ export function persistIngestBaselineIfClean(// empty is a real state and is persisted; without them, it is still treated// as discovery having gone wrong.if (mtimes.size === 0 && deletedFiles.size === 0) return false;- saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, replayedFiles);+ saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);return true;}@@ -1013,6 +900,76 @@ function renderProgressLine(phase: string, current: number, total: number): stri// Command registration// ---------------------------------------------------------------------------+type PathIngestOpts = Parameters<typeof ingestFiles>[1];++/** The workspace root `target` ingests into, or undefined when it cannot be resolved. */+function ingestLockRoot(target: string, explicitRoot?: string): string | undefined {+ try {+ const resolved = nodePath.resolve(target);+ return resolveIngestRoot(resolved, fs.statSync(resolved).isDirectory(), explicitRoot);+ } catch {+ // `ingestFiles` reports a missing or unreadable path itself.+ return undefined;+ }+}++/**+ * `ix ingest <path>`, single-flight per workspace under the lock `ix map`+ * holds. Two ingests (or an ingest and a map) over one workspace race on the+ * mtime baseline: the second to finish overwrites what the first recorded.+ *+ * A run that finds the lock held coalesces the way `ix map` does: it asks the+ * holder for one more pass (`requestMapRerun`) and exits 0, unless+ * IX_MAP_COALESCE_EXIT_CODE asks for another code. It used to exit 1, which+ * editor hooks and MCP clients that ingest after every edit reported as a+ * failed ingest. The holder takes the request before it lets go: `ix map`+ * re-ingests the workspace, and so does a holding `ix ingest` when the+ * workspace has a baseline -- an incremental pass, so it costs a stat per+ * file. Without a baseline that pass would be a first ingest of the whole+ * workspace, so it re-ingests only its own path then.+ */+export async function ingestPathSingleFlight(target: string, opts: PathIngestOpts): Promise<void> {+ const root = ingestLockRoot(target, opts.root);+ if (root === undefined) {+ await ingestFiles(target, opts);+ return;+ }+ const lock = acquireMapLock(root, `ix ingest ${root}`);+ if (lock === null) {+ requestMapRerun(root);+ if (opts.format === 'json') {+ // A JSON consumer gets an object, not an empty stdout it cannot parse.+ printJson({ coalesced: true, workspace: root });+ } else {+ process.stderr.write(chalk.dim(' Another ix map or ix ingest is running for this workspace; it will pick up this change before it exits.\n'));+ }+ applyRequestedMapCoalesceExitCode();+ return;+ }+ // Holding the lock: a request left before now is covered by this run, which+ // has not read the tree yet.+ takeMapRerun(root);+ try {+ await ingestFiles(target, opts);+ if (takeMapRerun(root)) {+ // One pass at most, as in `ix map`. The waiting run's own flags are not+ // known here, so this is a plain incremental pass.+ const rerunTarget = loadIngestBaseline(root) !== null ? root : target;+ try {+ await ingestFiles(rerunTarget, {+ recursive: true, format: 'json', printSummary: false, suppressOutput: true, root: opts.root,+ });+ } catch (err: any) {+ if (opts.format !== 'json') {+ process.stderr.write(chalk.dim(` Re-run for an ingest that waited on this one failed: ${err?.message ?? err}\n`));+ }+ }+ }+ } finally {+ lock.release();+ }+}+export function registerIngestCommand(program: Command): void {program.command('ingest [path]')@@ -1044,7 +1001,7 @@ export function registerIngestCommand(program: Command): void {if (opts.github) {await ingestGitHub(opts);} else if (effectivePath) {- await ingestFiles(effectivePath, opts);+ await ingestPathSingleFlight(effectivePath, opts);} else {console.error('Error: provide a <path> or use --github <owner/repo>');process.exit(1);@@ -1072,6 +1029,14 @@ const COMMIT_STATUS_IDEMPOTENT = 'Idempotent';*/const COMMIT_STATUS_BASE_REV_MISMATCH = 'BaseRevMismatch';+/** The stderr note for files skipped because their parse ran past the budget. */+export function describeParseTimeouts(files: readonly string[], sample = 5): string {+ const shown = files.slice(0, sample).join(", ");+ const more = files.length > sample ? ` and ${files.length - sample} more` : "";+ return `Skipped ${files.length} file(s) whose parse ran past the per-file budget: ${shown}${more}. ` ++ "They are not in the graph. Raise the budget with IX_PARSE_BUDGET_MS (milliseconds, 0 = none).";+}+/** Minimal local-ingest facts needed by commands that continue after ingestion. */export interface IngestFilesSummary {filesDiscovered: number;@@ -1158,6 +1123,12 @@ export interface IngestFilesSummary {* something must not claim the graph is where it was.*/graphUnchanged: boolean;+ /**+ * Files skipped because their parse ran past the per-file budget+ * (IX_PARSE_BUDGET_MS, default 10 s), workspace-relative. A subset of the+ * unparsed skips, named because a file that times out does so on every run.+ */+ parseTimeouts: string[];}/**@@ -1297,8 +1268,8 @@ export function describeStitchSkipped(// stay frozen. Only a run that re-ingests every file gets back in.// Every branch names `--force`, because nothing else re-registers this// workspace. A skipped stitch does NOT stop the run persisting its mtime- // baseline -- `persistIngestBaselineIfClean` gates on parse and commit errors- // only -- so an ordinary re-run is incremental, skips unchanged files, and+ // baseline -- the baseline records every file the run settled, stitch or+ // not -- so an ordinary re-run is incremental, skips unchanged files, and// never enters the stitch block. "Re-run once that finishes" and "raise// IX_MAP_DEADLINE_MS" were both promises this code cannot keep: the re-run// prints nothing, exits 0, and the cross-repo edges stay exactly as stale.@@ -1784,10 +1755,22 @@ export async function ingestFiles(* narrowing makes `pool` `never` down there.*/const crashedParses = (): number => (pool === null ? 0 : pool.crashedTasks());- const ensureParsePool = (): ParsePool => {- if (pool) return pool;- pool = new ParsePool(workerPath, Math.max(1, os.cpus().length - 1));- pool.init();+ /** Files whose parse ran past the budget; see `ParsePool.timedOutFiles`. */+ const timedOutParses = (): string[] => (pool === null ? [] : pool.timedOutFiles());+ /**+ * The pool, sized to the work the caller is about to give it: one worker per+ * 50 files, capped by the cores and by IX_PARSE_WORKERS (default 8). A+ * one-file edit used to start a worker per core -- 23 threads and about+ * 1.6 GB on a 24-core machine -- to parse one file. Workers start lazily, so+ * the size is a ceiling, and a later, larger phase can raise it.+ */+ const ensureParsePool = (expectedTasks: number): ParsePool => {+ const size = parsePoolSize(expectedTasks);+ if (pool) {+ pool.growTo(size);+ return pool;+ }+ pool = new ParsePool(workerPath, size);return pool;};@@ -1824,7 +1807,7 @@ export async function ingestFiles(// streaming loop below. A loss here is not a file the RUN lost, and// counting it as one refused the stitch and withheld the baseline over// files that were all present.- chunk.map(fp => ensureParsePool().parse(fp, relSources.get(fp)!, false).catch(() => null)),+ chunk.map(fp => ensureParsePool(targets.length).parse(fp, relSources.get(fp)!, false).catch(() => null)),);for (let j = 0; j < chunk.length; j++) {if (parsed[j]) preParsed.set(chunk[j], parsed[j]);@@ -1867,11 +1850,21 @@ export async function ingestFiles(* is told about is indistinguishable from an ingest that missed something.*/let filesExcluded = 0;+ /** What discovery left out: SKIPPED_DIRS hits and files it could not stat. */+ const discoveryCounts = emptyDiscoveryCounts();let filesChanged = 0;let patchesApplied = 0;let idempotentPatches = 0;/** Workspace-relative paths of changed files the backend answered `Idempotent` (F-01). */const replayedChanges = new Set<string>();+ // Per-file outcomes, so one failing file does not keep the whole baseline+ // from being written. A changed file is settled once its patch is committed+ // (`committedRel`), its content turns out to be what the backend has, or it+ // is skipped for a reason that will not change (minified); a file that+ // parsed to nothing is settled only if no worker crashed this run.+ const committedRel = new Set<string>();+ const settledAbs = new Set<string>();+ const unparsedRel = new Set<string>();// What `graphUnchanged` is computed from, beside the counters above. Each is// a way this run can have written to the graph, or have found it not where// the last run left it, without a patch being counted.@@ -2051,12 +2044,14 @@ export async function ingestFiles(const exclude = excludeMatcher.size > 0? { matcher: excludeMatcher, root: workspaceRoot, onSkip: () => { filesExcluded += 1; } }: undefined;+ // Discovery returns canonical paths, so confinement below does not resolve+ // every file a second time; only the root is canonicalized there.const discovery = discoverIngestFilePaths(stat.isFile()- ? (isSupportedSourceFile(scopePath) ? [scopePath] : [])- : (tryGitLsFiles(scopePath, opts.recursive ?? true, exclude)- ?? Array.from(walkFiles(scopePath, opts.recursive ?? true, exclude))),+ ? (isSupportedSourceFile(scopePath) ? [canonicalizeDiscoveredFilePath(scopePath)] : [])+ : discoverSourceFiles(scopePath, { recursive: opts.recursive ?? true, exclude, counts: discoveryCounts }),stat.isFile() ? undefined : scopePath,+ (p) => p,);// `--lang` narrows discovery to the named languages. The files it leaves// out keep their baseline entries (`langExcluded`, below): this run says@@ -2075,9 +2070,9 @@ export async function ingestFiles(// scope is read or parsed.const resolutionPaths = scopedRun? discoverIngestFilePaths(- tryGitLsFiles(workspaceRoot, true, exclude && { ...exclude, onSkip: undefined })- ?? Array.from(walkFiles(workspaceRoot, true, exclude && { ...exclude, onSkip: undefined })),+ discoverSourceFiles(workspaceRoot, { exclude: exclude && { ...exclude, onSkip: undefined } }),workspaceRoot,+ (p) => p,).files: filePaths;const resolveOpts = {@@ -2176,10 +2171,23 @@ export async function ingestFiles(// DB-reset guard: if the mtime cache has entries but the server returns no hashes// for a small sample, the DB was wiped (e.g. `ix reset` run from a different dir// so the cache wasn't cleared). Invalidate the cache so files are re-ingested.+ // Every baseline entry was committed (a file that failed keeps its previous+ // entry or none), so each sampled file should be known to the backend.+ // The sample is spread across the baseline rather than its first five+ // entries, and a lookup that FAILED proves nothing: it used to come back+ // empty and read as a reset, re-ingesting the whole workspace over one+ // timed-out request.if (!opts.force && mtimeCache.size > 0) {- const samplePaths = [...mtimeCache.keys()].slice(0, 5);- const sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug);- if (sampleHashes.size === 0) {+ const keys = [...mtimeCache.keys()];+ const step = Math.max(1, Math.floor(keys.length / 5));+ const samplePaths = keys.filter((_, i) => i % step === 0).slice(0, 5);+ let sampleHashes: Map<string, string> | undefined;+ try {+ sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug, true);+ } catch (err) {+ if (debug) process.stderr.write(`\n DB-reset check skipped: hash lookup failed: ${err}\n`);+ }+ if (sampleHashes !== undefined && sampleHashes.size === 0) {mtimeCache.clear();// What an unfinished re-ingest committed is gone with the rest.rebuildProgress?.clear();@@ -2364,6 +2372,7 @@ export async function ingestFiles(replayedChanges.add(item.filePath);return;}+ committedRel.add(item.filePath);onCommitted?.(item, rev);};@@ -2911,9 +2920,10 @@ export async function ingestFiles(// `BaseRevMismatch` wrote nothing: the backend read the latest rev// outside the transaction and it moved before the commit ran. It// is a lost race, not a replay, so it must not count as applied —- // and it belongs in commitErrors, because that is what stops- // `persistIngestBaselineIfClean` caching these files as unchanged- // and leaving them missing from the graph until the next edit.+ // it belongs in commitErrors, and it must not reach+ // `settleCommitted`: an unsettled file keeps its previous mtime,+ // which is what stops the baseline caching it as unchanged and+ // leaving it missing from the graph until the next edit.if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {commitErrors += items.length;if (debug) {@@ -3224,9 +3234,26 @@ export async function ingestFiles(}};- if ((knownHashes.size > 0 || mtimeChangedPaths.length === 0) && !opts.force) {+ // A local baseline with mtimes is enough to take the incremental path, even+ // when the backend knows none of the changed files: adding one new file to a+ // mapped repo is exactly that, and it used to re-send the whole repository.+ // `mtimeCache` is empty after --force, a workspace migration or the DB-reset+ // guard above, so those still take Path B.+ //+ // Only for files whose cross-file references Path A can resolve, though.+ // Path A resolves the changed files as one batch against `globalIndex`, and+ // that index holds the symbols of the prescan languages only (Go, PHP, R,+ // SAS, JS/TS). A new Python or Java file resolved that way binds its calls+ // to itself alone, so they point at a module node or at nothing; the+ // whole-repository pass resolves them. Those files keep that pass until the+ // persisted symbol table (IN-10) covers every language.+ const unknownToBackend = mtimeChangedPaths.filter(fp => !knownHashes.has(fp));+ const hasBaseline = previousBaseline !== null && mtimeCache.size > 0+ && unknownToBackend.every(needsIndexPrescan);+ if ((hasBaseline || knownHashes.size > 0 || mtimeChangedPaths.length === 0) && !opts.force) {// Path A: has baseline or all mtime-clean → pre-scan to detect changes before loading modules.- // If nothing changed, module load is skipped entirely.+ // If nothing changed, module load is skipped entirely. A file with no+ // backend hash is new: it is parsed and sent with no reconcile.const changedPaths: Array<{ filePath: string; bytes: Buffer; hash: string; previousHash: string | undefined }> = [];for (const filePath of mtimeChangedPaths) {try {@@ -3235,6 +3262,7 @@ export async function ingestFiles(if (!forceReingestPaths.has(filePath) && knownHashes.get(filePath) === hash) {filesSkipped++;filesSkippedAsUnchanged++;+ settledAbs.add(filePath);progressCurrent++;continue;}@@ -3263,6 +3291,7 @@ export async function ingestFiles(const sourceText = bytes.toString('utf-8');if (isLikelyMinifiedSource(sourceText)) {minifiedLikely++;+ settledAbs.add(filePath);filesSkipped++;progressCurrent++;if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);@@ -3319,13 +3348,13 @@ export async function ingestFiles(// paths. f.absFilePath is retained only for debug/error display.const parseResults = await Promise.all(chunk.map(f =>- ensureParsePool().parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),+ ensureParsePool(parseable.length).parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),),);const batch: ParsedFile[] = [];for (let j = 0; j < chunk.length; j++) {const parsed = parseResults[j] as any;- if (!parsed) { filesSkipped++; filesSkippedUnparsed++; continue; }+ if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(chunk[j].filePath); continue; }entitiesParsed += parsed.entities.length;batch.push({ filePath: chunk[j].filePath, parsed, hash: chunk[j].hash, previousHash: chunk[j].previousHash });}@@ -3335,18 +3364,17 @@ export async function ingestFiles(await pendingFlush;}} else {- // Path B: no baseline (first ingest) or --force → load modules, then stream parse + commit.+ // Path B: no baseline (first ingest, migrated workspace, wiped backend) or+ // --force → load modules, then stream parse + commit.//// Ix#568: this branch walks ALL of `filePaths`, re-reading, re-hashing and// re-parsing every one, and its hash-clean short-circuit cannot fire here// -- `knownHashes` is empty by construction on the non-force entry, and// the check is `!opts.force` on the other. So the mtime skips the stat- // loop recorded did not actually happen, and leaving them counted refused- // the stitch on the commonest incremental shape there is: add ONE new- // file to a mapped repo, and `loadExistingHashes` returns nothing for it,- // so `knownHashes.size === 0` sends the whole repo down here -- every file- // parsed, registration complete -- while the gate still saw N-1 files- // "skipped as unchanged" and silently declined to stitch.+ // loop recorded did not actually happen, and leaving them counted would+ // refuse the stitch on a run that parsed every file. (Adding one file to+ // a mapped repo used to land here whatever its language; the baseline+ // check above keeps prescan-language additions on Path A now.)filesSkipped -= filesSkippedAsUnchanged;filesSkippedAsUnchanged = 0;const moduleStart = performance.now();@@ -3466,12 +3494,14 @@ export async function ingestFiles(if (!opts.force && !forceReingestPaths.has(absFilePath) && knownHashes.get(absFilePath) === hash) {filesSkipped++;filesSkippedAsUnchanged++;+ settledAbs.add(absFilePath);return;}const previousHash = knownHashes.get(absFilePath);const sourceText = bytes.toString('utf-8');if (isLikelyMinifiedSource(sourceText)) {minifiedLikely++;+ settledAbs.add(absFilePath);filesSkipped++;if (debug) process.stderr.write(`\n [skip minified-likely] ${absFilePath}\n`);return;@@ -3479,7 +3509,7 @@ export async function ingestFiles(const relFilePath = toWorkspaceRelative(absFilePath);const fd: NonNullable<FileData> = { filePath: relFilePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };fileData[idx] = fd;- parsePromises[idx] = ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });+ parsePromises[idx] = ensureParsePool(filePaths.length).parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });} catch (err) {parseErrors++;process.stderr.write(`\n [read error] ${absFilePath}: ${err}\n`);@@ -3493,7 +3523,7 @@ export async function ingestFiles(const f = fileData[j];if (!f) continue;const parsed = parseResults[j] as any;- if (!parsed) { filesSkipped++; filesSkippedUnparsed++; continue; }+ if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(f.filePath); continue; }entitiesParsed += parsed.entities.length;batch.push({ filePath: f.filePath, parsed, hash: f.hash, previousHash: f.previousHash });}@@ -3570,17 +3600,26 @@ export async function ingestFiles(const recovery = pendingDeletionRecovery.get(item.filePath);if (!recovery || !previousBaseline) return;durableDeletedFiles.set(recovery.absolutePath, recovery.dependents);- saveIngestBaseline(- projectRoot,- previousMtimes,- previousBaseline.currentRev,- new Date(previousBaseline.lastIngestAt),- durableDeletedFiles,- previousBaseline.extractor,- previousBaseline.replayedFiles,- );},});+ // Once, after the deletions -- it was once per deleted file, the whole+ // baseline rewritten each time. The run's own baseline write follows;+ // this records the restore hints in case the run dies before then.+ if (previousBaseline && durableDeletedFiles.size > 0) {+ saveIngestBaseline(+ projectRoot,+ previousMtimes,+ previousBaseline.currentRev,+ new Date(previousBaseline.lastIngestAt),+ durableDeletedFiles,+ previousBaseline.extractor,+ {+ replayedFiles: previousBaseline.replayedFiles,+ pendingFiles: previousBaseline.pendingFiles,+ parseTimeouts: previousBaseline.parseTimeouts,+ },+ );+ }}}@@ -3618,33 +3657,67 @@ export async function ingestFiles(return abs !== undefined && (!inScope(abs) || langExcluded.includes(abs));}),];- const baselinePersisted = (scopedRun && !previousBaseline) ? false : persistIngestBaselineIfClean(+ // Per file, not all or nothing. The baseline used to be written only when+ // the whole run was clean, so one permanently unreadable file meant no run+ // ever recorded anything again: every map re-read the whole workspace and+ // deletions were never retracted. Now every file this run settled is+ // recorded, and every changed file it did not settle -- a read or build+ // error, a commit that failed, a parse lost to a dead worker -- keeps its+ // previous mtime (or none) so the next run tries it again. Lost parses+ // still count: a run whose worker pool died resolves every later file as+ // null, and recording those would skip them as unchanged for good.+ const runClean = ingestCompletedCleanly(parseErrors + crashedParses(), commitErrors);+ const lostParses = crashedParses() > 0;+ // A parse that ran past the budget is not settled either, though it is+ // not a crash: the file is not in the graph. It used to be recorded clean,+ // and since a slow file times out on every run, it was then skipped as+ // unchanged for good while status called the graph current.+ const timedOut = new Set(timedOutParses());+ const unsettled = mtimeChangedPaths.filter(abs => {+ if (settledAbs.has(abs)) return false;+ const rel = toWorkspaceRelative(abs);+ if (timedOut.has(rel)) return true;+ return !committedRel.has(rel) && !(unparsedRel.has(rel) && !lostParses);+ });+ // Name the unsettled files too. A new one has no previous mtime, so it is+ // left out of the baseline, and `ix status` judges a file the baseline+ // does not list against `lastIngestAt` -- which this write moves past the+ // file's mtime. Without the list, status called the graph current while+ // the file was missing from it. A replayed change has its own list. As+ // with replays, a run that did not look at a file has not resolved it.+ const notLookedAt = (rel: string): boolean => {+ const abs = nodePath.resolve(workspaceRoot, rel);+ return (!inScope(abs) || langExcluded.includes(abs)) && fs.existsSync(abs);+ };+ const pendingFiles = [+ ...unsettled.map(toWorkspaceRelative).filter(rel => !replayedChanges.has(rel) && !timedOut.has(rel)),+ ...(previousBaseline?.pendingFiles ?? []).filter(notLookedAt),+ ];+ // Timeouts get their own list, which `ix status` warns about by name: the+ // fix is a larger IX_PARSE_BUDGET_MS, not another run.+ const parseTimeouts = [...timedOut, ...(previousBaseline?.parseTimeouts ?? []).filter(notLookedAt)];+ // A deletion whose patch did not land keeps its entry, so it is found+ // deleted again next run.+ const failedDeletions = deletedPaths.filter(abs => !committedRel.has(toWorkspaceRelative(abs)));+ const baselinePersisted = (scopedRun && !previousBaseline) ? false : persistIngestBaseline(projectRoot,retryOnNextRun(- carryForwardMtimes(currentMtimes, previousMtimes, [...deferredDeletions, ...langExcluded, ...outOfScope]),+ carryForwardMtimes(currentMtimes, previousMtimes, [...deferredDeletions, ...failedDeletions, ...langExcluded, ...outOfScope]),previousMtimes,- replayedAbsolute,+ [...replayedAbsolute, ...unsettled],),latestRev,- // Lost parses count as parse errors HERE, whatever they are called- // elsewhere. A run whose worker pool died resolves every later file as- // null, and those increment `filesSkippedUnparsed` rather than- // `parseErrors` -- so without this the run wrote an mtime baseline for- // tens of thousands of files it never parsed or committed, exited 0, and- // every later incremental map skipped them as unchanged. Half the graph- // missing, recoverable only by `--force`, and worse than `main`, which- // respawns without a cap and loses one file per crash.- parseErrors + crashedParses(),- commitErrors,scopedRun && previousBaseline ? new Date(previousBaseline.lastIngestAt) : undefined,nextDeletedFiles,- baselineExtractor,- replayedFiles,+ // An unfinished re-ingest must not vouch for the new extractor: files+ // that failed would then be hash-clean next run and keep the old output.+ runClean ? baselineExtractor : (previousBaseline?.extractor ?? baselineExtractor),+ { replayedFiles, pendingFiles, parseTimeouts },);if (rebuildProgress !== null) {// Finished: the baseline now records the new extractor. Otherwise keep// what landed, so the next run resumes from here.- if (baselinePersisted) clearRebuildProgress(projectRoot);+ if (baselinePersisted && runClean) clearRebuildProgress(projectRoot);else saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress);}@@ -3912,6 +3985,7 @@ export async function ingestFiles(stitchErrors,stitchSkipped,stitchSkippedRule,+ parseTimeouts: timedOutParses(),graphUnchanged:opts.force !== true &&!workspaceMigrated &&@@ -3960,6 +4034,9 @@ export async function ingestFiles(// reads as an unexplained regression without the reason.process.stderr.write(` ${describeStitchSkipped(stitchSkipped, stitchSkippedRule, debug)}\n`);}+ if (summary.parseTimeouts.length > 0) {+ process.stderr.write(` ${describeParseTimeouts(summary.parseTimeouts)}\n`);+ }if (replayedChanges.size > 0) {// stderr and exit 0: nothing failed on this side, and the next run retries.process.stderr.write(` ${describeReplayedChanges(summary.replayedChanges)}\n`);@@ -4005,7 +4082,12 @@ export async function ingestFiles(// `minifiedLikely` and `unparsed` are the buckets that are subsets of// `filesSkipped`; `parseError` and `tooLarge` are counted separately and// always were.- skipReasons: { unchanged: filesSkippedAsUnchanged, emptyFile: filesSkippedAsEmpty, parseError: parseErrors + crashedParses(), unparsed: filesSkippedUnparsed, tooLarge, minifiedLikely, outsideRoot },+ skipReasons: {+ unchanged: filesSkippedAsUnchanged, emptyFile: filesSkippedAsEmpty, parseError: parseErrors + crashedParses(),+ unparsed: filesSkippedUnparsed, parseTimeout: summary.parseTimeouts.length, tooLarge, minifiedLikely, outsideRoot,+ unreadable: discoveryCounts.unreadable, skippedDirs: discoveryCounts.skippedDirs,+ },+ parseTimeouts: summary.parseTimeouts,commitErrors,stitchErrors,// Ix#568. Present only when the stitch was refused before it was sent, so@@ -4053,9 +4135,12 @@ export async function ingestFiles(if (replayedChanges.size > 0) console.log(` ${chalk.yellow('not applied:')} ${replayedChanges.size} ${chalk.dim('(see the warning above)')}`);if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);if (minifiedLikely > 0) console.log(` ${chalk.dim('skipped minified:')} ${minifiedLikely}`);+ if (summary.parseTimeouts.length > 0) console.log(` ${chalk.yellow("skipped parse timeout:")} ${summary.parseTimeouts.length} ${chalk.dim(`(${summary.parseTimeouts.join(", ")})`)}`);// Not dimmed like the others: these were dropped because the repo pointed// at files outside itself, which is worth a look rather than a shrug.if (outsideRoot > 0) console.log(` ${chalk.yellow('skipped outside root:')} ${outsideRoot} ${chalk.dim('(symlinks leaving the tree)')}`);+ if (discoveryCounts.unreadable > 0) console.log(` ${chalk.yellow("skipped unreadable:")} ${discoveryCounts.unreadable} ${chalk.dim("(listed but could not be read)")}`);+ if (discoveryCounts.skippedDirs > 0) console.log(` ${chalk.dim("skipped build/dependency dirs:")} ${discoveryCounts.skippedDirs}`);console.log(` rev: ${latestRev}`);if (patchesApplied === 0 && filesDiscovered === 0) {diff --git a/ix-cli/src/cli/commands/parse-pool.ts b/ix-cli/src/cli/commands/parse-pool.tsindex 44f5f0b9..529daecf 100644--- a/ix-cli/src/cli/commands/parse-pool.ts+++ b/ix-cli/src/cli/commands/parse-pool.ts@@ -26,8 +26,22 @@ type Task = {resolve: (result: unknown) => void;/** Does losing this one mean the RUN lost a file? See `parse`. */counts: boolean;+ /** The backstop for a parse that never answers; see `onTaskTimeout`. */+ timer?: ReturnType<typeof setTimeout>;};+/**+ * The per-file parse budget the worker enforces, read as core-ingestion's+ * `parseBudgetMs()` reads it: IX_PARSE_BUDGET_MS, default 10 s, 0 = none.+ * Restated rather than imported, because ix-cli does not depend on that+ * package at runtime.+ */+function parseBudgetMsFromEnv(): number {+ const raw = process.env.IX_PARSE_BUDGET_MS;+ const n = raw === undefined || raw.trim() === '' ? Number.NaN : Number(raw);+ return Number.isFinite(n) && n >= 0 ? n : 10_000;+}+export class ParsePool {private workers: Worker[] = [];private idle: Worker[] = [];@@ -39,21 +53,47 @@ export class ParsePool {* stops waiting for it.* @param maxWaitMs The ceiling on waiting for a BUSY one.*- * Both injectable only so the tests can pin the behaviour without spending- * the real periods -- the ceiling test would otherwise cost ten seconds on- * every leg of the matrix. `ingestFiles` passes neither.+ * @param taskTimeoutMs How long one parse may hold a worker before the pool+ * gives up on it; 0 = never. See `onTaskTimeout`.+ *+ * All three injectable only so the tests can pin the behaviour without+ * spending the real periods -- the ceiling test would otherwise cost ten+ * seconds on every leg of the matrix. `ingestFiles` passes none of them.*/constructor(private workerPath: string,private concurrency: number,private graceMs: number = ParsePool.SHUTDOWN_GRACE_MS,private maxWaitMs: number = ParsePool.SHUTDOWN_MAX_WAIT_MS,+ private taskTimeoutMs: number = 2 * parseBudgetMsFromEnv(),) {}+ /**+ * Start every worker now. Optional: without it the pool starts workers as+ * work queues up (see `drain`), which is what `ingestFiles` wants -- a+ * one-file edit then starts one worker, not `concurrency` of them.+ */init(): void {- for (let i = 0; i < this.concurrency; i++) {- this.spawnWorker();- }+ while (this.started < this.concurrency) this.startWorker();+ }++ /**+ * Raise the ceiling on workers. A later, larger phase of a run asks for more+ * than the first one did; the pool never shrinks, and the new workers start+ * only if queued work needs them.+ */+ growTo(concurrency: number): void {+ if (concurrency <= this.concurrency) return;+ this.concurrency = concurrency;+ this.drain();+ }++ /** Workers started to fill the ceiling. Replacements for dead ones are not counted; see `onError`. */+ private started = 0;++ private startWorker(): void {+ this.started++;+ this.spawnWorker();}/**@@ -132,7 +172,10 @@ export class ParsePool {// Not counted, for the same reason the stranded queue is not: every gate// that reads `crashedParses()` has already run by the time `destroy()` is// called from the outermost `finally`.- for (const task of this.active.values()) task.resolve(null);+ for (const task of this.active.values()) {+ clearTimeout(task.timer);+ task.resolve(null);+ }this.active.clear();this.workers = [];this.idle = [];@@ -421,12 +464,77 @@ export class ParsePool {}private drain(): void {+ // Start workers on demand, one per queued task no idle worker can take,+ // up to the ceiling. Only first starts: a dead worker's replacement is+ // `onError`'s, under the respawn cap, so a worker that dies on load cannot+ // turn this into a spawn loop.+ while (!this.dead && !this.destroyed && this.queue.length > this.idle.length && this.started < this.concurrency) {+ this.startWorker();+ }while (this.idle.length > 0 && this.queue.length > 0) {const w = this.idle.pop()!;const task = this.queue.shift()!;this.active.set(w, task);w.postMessage({ filePath: task.filePath, source: task.source });+ if (this.taskTimeoutMs > 0) {+ task.timer = setTimeout(() => this.onTaskTimeout(w, task), this.taskTimeoutMs);+ // A pending backstop must not hold the process open on its own.+ task.timer.unref?.();+ }+ }+ }++ /**+ * Files whose parse ran past the budget, in the order they gave up --+ * reported by the worker's own budget or by the backstop below. Only tasks+ * that count (see `parse`). Not counted in `crashedTasks()`: a crash leaves+ * every file the run could not parse unsettled in the baseline, and one slow+ * file must not do that to the rest. The ingest keeps just these files+ * unsettled and records them for `ix status`.+ */+ timedOutFiles(): string[] {+ return [...this.timedOut];+ }++ private timedOut: string[] = [];++ /**+ * The backstop for a parse that never answers.+ *+ * The worker enforces the parse budget itself, but only between matches: a+ * file stuck inside tree-sitter's native parse or query never reaches a+ * check. So past twice the budget the pool settles the task as a timeout,+ * lets the worker go and starts a replacement under the same respawn cap a+ * crash uses.+ *+ * Let go, NOT terminated -- `shutdown` explains at length why `terminate()`+ * segfaults the process. The thread is asked to close once it is done and is+ * `unref`'d meanwhile, so it burns its core until the native call returns or+ * the process exits, and its late answer goes nowhere.+ */+ private onTaskTimeout(w: Worker, task: Task): void {+ if (this.destroyed || this.active.get(w) !== task) return;+ this.active.delete(w);+ if (task.counts) this.timedOut.push(task.filePath);+ task.resolve(null);+ const idx = this.workers.indexOf(w);+ if (idx !== -1) this.workers.splice(idx, 1);+ // As `shutdown` does when it gives up, and for the same reasons: drop the+ // listeners that close over the pool, keep one that captures nothing.+ w.removeAllListeners('message');+ w.removeAllListeners('exit');+ w.removeAllListeners('error');+ w.on('error', SWALLOW_ERROR);+ w.postMessage({ __shutdown: true });+ w.unref();+ if (this.respawns < ParsePool.MAX_RESPAWNS) {+ this.respawns++;+ this.spawnWorker();+ } else if (this.workers.length === 0) {+ this.latchDead();+ return;}+ this.drain();}/**@@ -513,10 +621,12 @@ export class ParsePool {return this.deaths;}- private onResult(w: Worker, msg: { ok: boolean; result: unknown }): void {+ private onResult(w: Worker, msg: { ok: boolean; result: unknown; reason?: string }): void {const task = this.active.get(w);if (!task) return;this.active.delete(w);+ clearTimeout(task.timer);+ if (!msg.ok && msg.reason === 'timeout' && task.counts) this.timedOut.push(task.filePath);// A successful round trip clears the respawn budget. The cap exists to// stop a worker that dies deterministically from spinning spawn -> die ->// spawn; it is not meant to be a lifetime quota. As a per-run total it made@@ -579,6 +689,7 @@ export class ParsePool {const task = this.active.get(w);if (task) {this.active.delete(w);+ clearTimeout(task.timer);if (task.counts) this.crashed++;task.resolve(null); // isolate: failed file = null parse result}@@ -627,19 +738,20 @@ export class ParsePool {// none, the `else if` latches `dead`, strands the queue and resolves it// as crashed, then returns -- so that path never reaches `drain()`. With// others still alive, NEITHER branch body runs and control falls straight- // through to `drain()`. `ingest.ts` builds the pool with- // `Math.max(1, os.cpus().length - 1)` and `respawns` is a pool-wide budget- // that any success resets, so on an ordinary machine the fall-through is- // the usual case -- do not read the cap as implying the pool is finished.+ // through to `drain()`. `ingest.ts` sizes the pool with `parsePoolSize`+ // -- one worker per 50 files, at most the cores less one or 8 -- and+ // `respawns` is a pool-wide budget that any success resets, so on a run+ // of any size the fall-through is the usual case -- do not read the cap as+ // implying the pool is finished.//- // The floor in that expression is there because 1 is reachable, and at- // concurrency 1 this inverts: the first death PAST the cap is also the last+ // Concurrency 1 is common -- any run of 50 files or fewer -- and there+ // this inverts: the first death PAST the cap is also the last// worker, so `workers.length === 0` always holds and the pool always// latches `dead`. (Not the death that exhausts the budget -- that one// still takes the respawn branch and spawns a replacement. The two are// one apart, and this file is where that distinction has to stay- // straight.) Above concurrency 1 it does not invert. Which case a given machine falls- // into depends on its core count, and this comment deliberately does not+ // straight.) Above concurrency 1 it does not invert. Which case a given run falls+ // into depends on its size and the core count, and this comment deliberately does not// say -- an earlier revision guessed at CI's and contradicted the machine// sizes in `docs/parse-pool-teardown.md`, which is the file that owns// them.@@ -649,18 +761,24 @@ export class ParsePool {this.respawns++;this.spawnWorker();} else if (this.workers.length === 0) {- // Out of workers and out of replacements. Nothing queued can ever be- // parsed, so resolve it rather than leave `Promise.all` waiting on a pool- // that no longer exists -- and LATCH it, because `parse()` must answer the- // same way for every call after this, not just for what happened to be- // queued at this instant. Counted as crashed, which is what it is, so the- // stitch gate knows this run lost files.- this.dead = true;- const stranded = this.queue.splice(0, this.queue.length);- this.crashed += stranded.filter(t => t.counts).length;- for (const t of stranded) t.resolve(null);+ this.latchDead();return;}this.drain();}++ /**+ * Out of workers and out of replacements. Nothing queued can ever be+ * parsed, so resolve it rather than leave `Promise.all` waiting on a pool+ * that no longer exists -- and LATCH it, because `parse()` must answer the+ * same way for every call after this, not just for what happened to be+ * queued at this instant. Counted as crashed, which is what it is, so the+ * stitch gate knows this run lost files.+ */+ private latchDead(): void {+ this.dead = true;+ const stranded = this.queue.splice(0, this.queue.length);+ this.crashed += stranded.filter(t => t.counts).length;+ for (const t of stranded) t.resolve(null);+ }}diff --git a/ix-cli/src/cli/commands/status.ts b/ix-cli/src/cli/commands/status.tsindex e3f6002d..bcd5eeeb 100644--- a/ix-cli/src/cli/commands/status.ts+++ b/ix-cli/src/cli/commands/status.ts@@ -20,6 +20,8 @@ interface StatusStaleInfo {sampleChangedFiles: string[];/** Changed files the last run could not get applied (F-01); see `IngestBaseline.replayedFiles`. */replayedFiles?: string[];+ /** Files skipped because their parse ran past the budget; see `IngestBaseline.parseTimeouts`. */+ parseTimeouts?: string[];}/**@@ -47,8 +49,10 @@ export function renderStatusLlm(["last_ingest_at", staleInfo?.lastIngestAt ?? null],["stale_files", staleInfo ? String(staleInfo.staleFiles) : null],["not_applied", staleInfo ? String(staleInfo.replayedFiles?.length ?? 0) : null],+ ["parse_timeouts", staleInfo ? String(staleInfo.parseTimeouts?.length ?? 0) : null],["stale", staleInfo- ? (!staleInfo.graphCompleted || staleInfo.staleFiles > 0 || (staleInfo.replayedFiles?.length ?? 0) > 0 ? "true" : "false")+ ? (!staleInfo.graphCompleted || staleInfo.staleFiles > 0 || (staleInfo.replayedFiles?.length ?? 0) > 0+ || (staleInfo.parseTimeouts?.length ?? 0) > 0 ? "true" : "false"): null],])];for (const f of staleInfo?.sampleChangedFiles ?? []) {@@ -57,9 +61,20 @@ export function renderStatusLlm(for (const f of staleInfo?.replayedFiles ?? []) {lines.push(llmLine("not_applied", [["path", f]]));}+ for (const f of staleInfo?.parseTimeouts ?? []) {+ lines.push(llmLine("parse_timeout", [["path", f]]));+ }return lines;}+/** The `ix status` warning for files the last run skipped on the parse budget. */+function describeStatusParseTimeouts(files: readonly string[], sample = 5): string {+ const shown = files.slice(0, sample).join(", ");+ const more = files.length > sample ? ` and ${files.length - sample} more` : "";+ return `${files.length} file(s) are not in the graph: their parse ran past the per-file budget (${shown}${more}). ` ++ "Raise IX_PARSE_BUDGET_MS (milliseconds, 0 = none) and run ix map.";+}+export function registerStatusCommand(program: Command): void {program.command("status")@@ -92,6 +107,7 @@ export function registerStatusCommand(program: Command): void {staleFiles: staleInfo?.staleFiles ?? 0,sampleChangedFiles: staleInfo?.sampleChangedFiles ?? [],replayedFiles: staleInfo?.replayedFiles ?? [],+ parseTimeouts: staleInfo?.parseTimeouts ?? [],};printJson(result);} else {@@ -120,9 +136,14 @@ export function registerStatusCommand(program: Command): void {renderNote(`... and ${staleInfo.staleFiles - staleInfo.sampleChangedFiles.length} more`);}renderNote("Run ix map to update.");- } else {+ } else if (staleInfo.parseTimeouts.length === 0) {renderSuccess("Graph is up to date.");}+ if (staleInfo.graphCompleted && staleInfo.parseTimeouts.length > 0) {+ // Beside the other states, not instead of them: another map+ // will not bring these files in, a larger budget will.+ renderWarning(describeStatusParseTimeouts(staleInfo.parseTimeouts));+ }if (staleInfo.graphCompleted && !staleInfo.mapCompleted) {renderWarning("No completed architecture map is recorded for this source revision.");renderNote("Source graph reads remain available, but hierarchy views may be incomplete.");diff --git a/ix-cli/src/cli/commands/watch.ts b/ix-cli/src/cli/commands/watch.tsindex ae1a7ea4..6ef60790 100644--- a/ix-cli/src/cli/commands/watch.ts+++ b/ix-cli/src/cli/commands/watch.ts@@ -8,19 +8,8 @@ import type { Command } from "commander";import chalk from "chalk";import { resolveWorkspaceRoot, clearIngestMtimeCache } from "../config.js";import { bootstrap, ensureWorkspaceIdState } from "../bootstrap.js";-import { SUPPORTED_EXTENSIONS } from "../supported-extensions.js";+import { discoverSourceFiles, isGeneratedFile, isSupportedSourceFile, SKIPPED_DIRS } from "../file-discovery.js";-const SUPPORTED_NAMES = new Set([- ".gitignore", ".gitattributes", ".editorconfig", ".env",- ".eslintrc", ".prettierrc", ".babelrc",- "Makefile", "Dockerfile", "Procfile", "Gemfile", "Rakefile",- "BUILD", "WORKSPACE",-]);--const IGNORE_DIRS = new Set([- "node_modules", ".git", "dist", "build", "target", ".next",- ".cache", "__pycache__", ".ix", ".claude",-]);const DEBOUNCE_MS = 300;const MAP_COALESCED_EXIT_CODE = 75;@@ -216,15 +205,15 @@ export function updatePollingSnapshot(return changed;}+/** The files `ix map` would ingest -- the same test discovery applies. */function isSupportedPath(filePath: string): boolean {- const ext = path.extname(filePath).toLowerCase();- return SUPPORTED_EXTENSIONS.has(ext) || SUPPORTED_NAMES.has(path.basename(filePath));+ return isSupportedSourceFile(filePath) && !isGeneratedFile(path.basename(filePath));}export function shouldWatch(root: string, filePath: string): boolean {if (!isSupportedPath(filePath)) return false;const segments = path.relative(root, filePath).split(path.sep);- return !segments.some(segment => IGNORE_DIRS.has(segment));+ return !segments.some(segment => SKIPPED_DIRS.has(segment));}export function prepareMigratedWorkspaceRefresh(@@ -234,23 +223,9 @@ export function prepareMigratedWorkspaceRefresh(clear(root);}+/** The files `ix map` would discover under `dir` (see `file-discovery.ts`). */function collectFiles(dir: string): string[] {- const results: string[] = [];- const stack = [dir];- while (stack.length > 0) {- const current = stack.pop()!;- let entries: fs.Dirent[];- try { entries = fs.readdirSync(current, { withFileTypes: true }); }- catch { continue; }- for (const entry of entries) {- if (IGNORE_DIRS.has(entry.name)) continue;- if (entry.name.startsWith(".") && entry.isDirectory()) continue;- const fullPath = path.join(current, entry.name);- if (entry.isDirectory()) stack.push(fullPath);- else if (entry.isFile() && isSupportedPath(fullPath)) results.push(fullPath);- }- }- return results;+ return discoverSourceFiles(dir);}function createBatchNotifier(root: string, scheduler: WatchRefreshScheduler): {diff --git a/ix-cli/src/cli/file-discovery.ts b/ix-cli/src/cli/file-discovery.tsnew file mode 100644index 00000000..2e4dccb1--- /dev/null+++ b/ix-cli/src/cli/file-discovery.ts@@ -0,0 +1,303 @@+// Copyright 2026 Ix Infrastructure Inc.++import * as fs from "node:fs";+import * as nodePath from "node:path";+import { spawnSync } from "node:child_process";++import { createIgnoreMatcher, parseIgnoreFile, type IgnoreMatcher } from "./ignore-globs.js";+import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";++/**+ * Which files are the repository: one implementation for `ix ingest`/`ix map`,+ * `ix watch` and `ix status`.+ *+ * There were three, and they disagreed. The filesystem walk skipped `tests/`,+ * `examples/`, `fixtures/`, Go's `pkg/` and every dot-directory, while+ * `git ls-files` -- the path in any git repository -- skipped none of them, so+ * the same tree gave a different graph depending on whether it was a git+ * checkout. `watch` and `status` had their own lists again. And `git ls-files`+ * quoted any non-ASCII name (`"caf\303\251.ts"`), the stat on that string+ * failed, and the file was silently dropped.+ *+ * Now: git lists the files when it can (NUL-separated, unquoted); otherwise the+ * walk honours each directory's `.gitignore`, so the two agree on an ordinary+ * tree. The walk also skips a short list of directories that are never source+ * (build output, dependencies, VCS metadata, caches) by name. In a work tree+ * git's own ignore rules decide instead: a tracked file is kept whatever its+ * directory is called -- a committed `build/` or `out/` can be source -- and+ * only untracked files under those names are skipped. Both count what they+ * skipped. Tests and fixtures are in the set; the role classifier is what+ * tells them apart.+ *+ * The walk's `.gitignore` reading is the `.ixignore` subset (see+ * `ignore-globs.ts`): no `!` negation, so it can exclude a little more than+ * git does.+ */++/**+ * Never source: build output, dependencies, VCS metadata, tool caches. Skipped+ * by name in a walk, and for untracked files in a work tree; never for a file+ * git tracks.+ */+export const SKIPPED_DIRS: ReadonlySet<string> = new Set([+ "node_modules", ".git", ".hg", ".svn", "dist", "build", "target", "out", ".next",+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",+ ".gradle", ".idea", ".vscode", ".settings", ".vs",+ ".cache", ".parcel-cache", "coverage", ".nyc_output",+ "Pods", ".dart_tool", ".pub-cache", "obj",+ ".terraform", ".serverless", ".turbo", ".svelte-kit", ".nuxt", ".angular", ".expo",+ ".ix", ".claude", ".gitnexus",+]);++// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.+// These files inflate parse time and graph size without contributing to architectural signal.+const GENERATED_FILE_SUFFIXES = [+ ".pb.go", // protobuf generated Go+ "_deepcopy.go", // controller-gen deepcopy (Kubernetes ecosystem)+ "_mock.go", // gomock generated mocks+ ".pb.ts", // protobuf generated TypeScript+ ".pb.js", // protobuf generated JavaScript+ "_pb.ts", // alternative protobuf TypeScript naming+ "_pb.js", // alternative protobuf JavaScript naming+];++const GENERATED_FILE_PREFIXES = [+ "zz_generated", // controller-gen convention (Kubernetes ecosystem)+ "mock_", // gomock convention+];++// Generated dependency lockfiles, matched whole. Only the ones whose extension+// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,+// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end+// in `.lock`, which is not a discovered extension, so they never reach here.+// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger+// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and+// it describes a resolved dependency snapshot, not architecture.+const GENERATED_FILE_NAMES = new Set([+ "package-lock.json", // npm+ "npm-shrinkwrap.json", // npm, published form+ "pnpm-lock.yaml", // pnpm+ "packages.lock.json", // NuGet+]);++export function isGeneratedFile(basename: string): boolean {+ // Lowercased for the name match alone: `isSupportedSourceFile` lowercases+ // too, so on a case-insensitive filesystem a `Package-lock.json` is+ // discovered and would otherwise slip past a case-sensitive comparison.+ if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;+ for (const suffix of GENERATED_FILE_SUFFIXES) {+ if (basename.endsWith(suffix)) return true;+ }+ for (const prefix of GENERATED_FILE_PREFIXES) {+ if (basename.startsWith(prefix)) return true;+ }+ return false;+}++export function isSupportedSourceFile(filePath: string): boolean {+ const fileName = nodePath.basename(filePath).toLowerCase();+ return fileName === "dockerfile"+ || fileName.endsWith(".dockerfile")+ || fileName === "makefile"+ || fileName === "gnumakefile"+ || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());+}++/** A file discovery would hand to the parser. */+function isCandidate(filePath: string): boolean {+ return isSupportedSourceFile(filePath) && !isGeneratedFile(nodePath.basename(filePath));+}++export type DiscoveryExclude = { matcher: IgnoreMatcher; root: string; onSkip?: () => void };++/** What discovery left out, so it can be reported rather than guessed at. */+export interface DiscoveryCounts {+ /** Untracked files under a `SKIPPED_DIRS` directory (git mode) or such directories (walk). */+ skippedDirs: number;+ /** Listed files that could not be stat'd: permissions, a race with a delete. */+ unreadable: number;+}++export function emptyDiscoveryCounts(): DiscoveryCounts {+ return { skippedDirs: 0, unreadable: 0 };+}++export function canonicalizeDiscoveredFilePath(filePath: string): string {+ try {+ return fs.realpathSync.native(filePath);+ } catch {+ return nodePath.resolve(filePath);+ }+}++function inSkippedDir(relativePath: string): boolean {+ const parts = relativePath.split(/[/\\]/);+ parts.pop();+ return parts.some(p => SKIPPED_DIRS.has(p));+}++const GIT_SAFE_CONFIG = [+ "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.quotePath=false",+];++/**+ * The files `git ls-files` reports under `dir`, canonical and de-duplicated, or+ * null when `dir` is not in a git work tree.+ *+ * `-z` and `core.quotePath=false`: without them git quotes a non-ASCII name+ * (`"caf\303\251.ts"`), the stat on that string fails, and the file vanished.+ */+export function tryGitLsFiles(+ dir: string,+ recursive: boolean,+ exclude?: DiscoveryExclude,+ counts: DiscoveryCounts = emptyDiscoveryCounts(),+): string[] | null {+ try {+ const result = spawnSync(+ "git",+ // core.fsmonitor / core.hooksPath: a cloned repository's own .git/config+ // can name a command ls-files would run. Never let it.+ [...GIT_SAFE_CONFIG, "ls-files", "-z", "--cached", "--others", "--exclude-standard"],+ {+ cwd: dir,+ encoding: "utf-8",+ maxBuffer: 64 * 1024 * 1024,+ },+ );+ if (result.status !== 0) return null;++ // Asked only when a listed file sits under a skipped name.+ let untracked: Set<string> | null = null;+ const isUntracked = (entry: string): boolean => {+ untracked ??= gitUntracked(dir);+ return untracked.has(entry);+ };+ const files: string[] = [];+ const seenCanonical = new Set<string>();+ for (const entry of result.stdout.split("\0")) {+ if (!entry) continue;+ const fullPath = nodePath.resolve(dir, entry);+ if (!recursive && nodePath.dirname(fullPath) !== dir) continue;+ // Before the extension check, so the count reports what the caller+ // excluded rather than what they excluded and Ix would have skipped+ // anyway.+ if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {+ exclude.onSkip?.();+ continue;+ }+ if (!isCandidate(fullPath)) continue;+ // A tracked file stays: git is told what is generated through+ // .gitignore, and a repository that commits a `build/` or `out/` tree+ // is saying it is source. An untracked one under such a name is what a+ // missing .gitignore line would have caught -- a fresh `node_modules`.+ if (inSkippedDir(entry) && isUntracked(entry)) { counts.skippedDirs++; continue; }+ try {+ if (!fs.statSync(fullPath).isFile()) continue;+ } catch {+ counts.unreadable++;+ continue;+ }+ const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);+ if (seenCanonical.has(canonicalPath)) continue;+ seenCanonical.add(canonicalPath);+ files.push(canonicalPath);+ }+ return files;+ } catch {+ return null;+ }+}++/** Untracked, not-ignored entries under `dir`, as `git ls-files` spells them. */+function gitUntracked(dir: string): Set<string> {+ const result = spawnSync("git", [...GIT_SAFE_CONFIG, "ls-files", "-z", "--others", "--exclude-standard"], {+ cwd: dir,+ encoding: "utf-8",+ maxBuffer: 64 * 1024 * 1024,+ });+ // Unknown: treat every listed file as tracked, which keeps it.+ if (result.status !== 0) return new Set();+ return new Set(result.stdout.split("\0").filter(Boolean));+}++/** `.gitignore` matchers in force for a directory: each one's own, with the directory it is relative to. */+type GitignoreStack = Array<{ dir: string; matcher: IgnoreMatcher }>;++function withGitignore(dir: string, stack: GitignoreStack): GitignoreStack {+ try {+ const patterns = parseIgnoreFile(fs.readFileSync(nodePath.join(dir, ".gitignore"), "utf-8"));+ if (patterns.length > 0) return [...stack, { dir, matcher: createIgnoreMatcher(patterns) }];+ } catch {+ // No .gitignore here, the ordinary case.+ }+ return stack;+}++function gitignored(stack: GitignoreStack, fullPath: string, isDirectory: boolean): boolean {+ return stack.some(({ dir, matcher }) => matcher.matches(nodePath.relative(dir, fullPath), isDirectory));+}++/**+ * Walk `dir` for candidate files, honouring every `.gitignore` on the way down+ * and skipping `SKIPPED_DIRS`. Yields paths as found; `discoverSourceFiles`+ * canonicalizes them.+ */+export function* walkFiles(+ dir: string,+ recursive: boolean,+ exclude?: DiscoveryExclude,+ counts: DiscoveryCounts = emptyDiscoveryCounts(),+ gitignores: GitignoreStack = [],+): Generator<string> {+ let entries: fs.Dirent[];+ try { entries = fs.readdirSync(dir, { withFileTypes: true }); }+ catch { return; }+ const stack = withGitignore(dir, gitignores);++ for (const entry of entries) {+ const isDirectory = entry.isDirectory();+ if (isDirectory && SKIPPED_DIRS.has(entry.name)) { counts.skippedDirs++; continue; }+ const full = nodePath.join(dir, entry.name);+ // Directories are tested before descending, so an excluded tree costs one+ // check rather than one per file inside it.+ if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), isDirectory)) {+ exclude.onSkip?.();+ continue;+ }+ if (gitignored(stack, full, isDirectory)) continue;+ if (isDirectory) {+ if (recursive) yield* walkFiles(full, true, exclude, counts, stack);+ } else if (entry.isFile()) {+ if (isCandidate(entry.name)) yield full;+ }+ }+}++/**+ * The source files under `dir`, canonical and de-duplicated: git's list when+ * `dir` is in a work tree, else the walk.+ *+ * `walkLimit` stops the walk after that many files. A git listing is bounded+ * by the repository, but a walk is bounded by nothing: a command run from a+ * home directory would read the whole disk.+ */+export function discoverSourceFiles(+ dir: string,+ opts: { recursive?: boolean; exclude?: DiscoveryExclude; counts?: DiscoveryCounts; walkLimit?: number } = {},+): string[] {+ const recursive = opts.recursive ?? true;+ const counts = opts.counts ?? emptyDiscoveryCounts();+ const listed = tryGitLsFiles(dir, recursive, opts.exclude, counts);+ if (listed) return listed;+ const out: string[] = [];+ const seen = new Set<string>();+ for (const filePath of walkFiles(dir, recursive, opts.exclude, counts)) {+ if (opts.walkLimit !== undefined && out.length >= opts.walkLimit) break;+ const canonical = canonicalizeDiscoveredFilePath(filePath);+ if (seen.has(canonical)) continue;+ seen.add(canonical);+ out.push(canonical);+ }+ return out;+}diff --git a/ix-cli/src/cli/ingest-baseline.ts b/ix-cli/src/cli/ingest-baseline.tsindex d44e4d8a..df72df96 100644--- a/ix-cli/src/cli/ingest-baseline.ts+++ b/ix-cli/src/cli/ingest-baseline.ts@@ -13,6 +13,8 @@ interface SerializedIngestBaseline {tracksMapBaseline?: boolean;extractor?: string;replayedFiles?: string[];+ pendingFiles?: string[];+ parseTimeouts?: string[];}export interface IngestBaseline {@@ -46,8 +48,40 @@ export interface IngestBaseline {* are kept at the previous value, so the next run sends them again.*/replayedFiles: string[];+ /**+ * Workspace-relative paths of changed files the last run could not ingest:+ * a read or build error, a failed commit, a parse lost to a dead worker.+ * Each keeps its previous mtime, so the next run retries it. A new file has+ * no previous mtime and is left out of `files`, and a file missing from+ * `files` is judged stale against `lastIngestAt`, which that same run moved+ * past the file's mtime -- so without this list `ix status` called the+ * graph current while the file was not in it.+ */+ pendingFiles: string[];+ /**+ * Workspace-relative paths of files whose parse ran past the per-file+ * budget (`IX_PARSE_BUDGET_MS`) on the last run, so they are not in the+ * graph. Not settled: like `pendingFiles` they keep their previous mtime+ * (or none) and every run tries them again, and `ix status` warns about+ * them rather than calling the graph current. A slow file times out on+ * every run, so recording it clean skipped it for good.+ */+ parseTimeouts: string[];+}++/** Per-file lists a baseline write records beside the mtimes. */+export interface BaselineFileNotes {+ /** See `IngestBaseline.replayedFiles`. */+ replayedFiles?: readonly string[];+ /** See `IngestBaseline.pendingFiles`. */+ pendingFiles?: readonly string[];+ /** See `IngestBaseline.parseTimeouts`. */+ parseTimeouts?: readonly string[];}+const stringList = (value: unknown): string[] =>+ Array.isArray(value) ? value.filter((p): p is string => typeof p === "string") : [];+/*** What counts as a revision, for both sides of this file.*@@ -93,9 +127,9 @@ export function loadIngestBaseline(projectRoot: string): IngestBaseline | null {lastIngestAt,tracksMapBaseline: data.tracksMapBaseline === true,extractor: typeof data.extractor === "string" ? data.extractor : null,- replayedFiles: Array.isArray(data.replayedFiles)- ? data.replayedFiles.filter((p): p is string => typeof p === "string")- : [],+ replayedFiles: stringList(data.replayedFiles),+ pendingFiles: stringList(data.pendingFiles),+ parseTimeouts: stringList(data.parseTimeouts),};} catch {return null;@@ -109,7 +143,7 @@ export function saveIngestBaseline(now: Date = new Date(),deletedFiles: Map<string, string[]> = new Map(),extractor?: string | null,- replayedFiles: readonly string[] = [],+ { replayedFiles = [], pendingFiles = [], parseTimeouts = [] }: BaselineFileNotes = {},): void {try {// Keep the last good rev rather than writing a shape the read side will@@ -134,9 +168,23 @@ export function saveIngestBaseline(tracksMapBaseline: true,...(extractor ? { extractor } : {}),...(replayedFiles.length > 0 ? { replayedFiles: [...replayedFiles].sort() } : {}),+ ...(pendingFiles.length > 0 ? { pendingFiles: [...new Set(pendingFiles)].sort() } : {}),+ ...(parseTimeouts.length > 0 ? { parseTimeouts: [...new Set(parseTimeouts)].sort() } : {}),};- fs.mkdirSync(path.dirname(ingestMtimeCachePath(projectRoot)), { recursive: true });- fs.writeFileSync(ingestMtimeCachePath(projectRoot), JSON.stringify(data));+ const target = ingestMtimeCachePath(projectRoot);+ fs.mkdirSync(path.dirname(target), { recursive: true });+ // Via a temp file and rename: a process killed mid-write left a truncated+ // baseline, which reads as none, and the next map re-ingested everything.+ const tmp = `${target}.${process.pid}.tmp`;+ try {+ fs.writeFileSync(tmp, JSON.stringify(data));+ fs.renameSync(tmp, target);+ } catch (err) {+ // A failed rename (a locked target on Windows, say) must not leave the+ // temp file behind; one would pile up per failing run.+ fs.rmSync(tmp, { force: true });+ throw err;+ }} catch {// The cache is an optimization and freshness hint. Ingestion itself succeeded.}diff --git a/ix-cli/src/cli/stale.ts b/ix-cli/src/cli/stale.tsindex c787efaf..3422dd16 100644--- a/ix-cli/src/cli/stale.ts+++ b/ix-cli/src/cli/stale.ts@@ -5,7 +5,7 @@ import * as path from "node:path";import { resolveWorkspaceRoot } from "./config.js";import { loadIngestBaseline } from "./ingest-baseline.js";import { hasCompletedMapFor } from "./map-baseline.js";-import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";+import { canonicalizeDiscoveredFilePath, discoverSourceFiles } from "./file-discovery.js";export interface StaleInfo {graphCompleted: boolean;@@ -16,53 +16,30 @@ export interface StaleInfo {sampleChangedFiles: string[];/** Changed files the last run could not get applied (F-01); see `IngestBaseline.replayedFiles`. */replayedFiles: string[];+ /**+ * Files the last run skipped because their parse ran past the budget, still+ * on disk; see `IngestBaseline.parseTimeouts`. Not counted in `staleFiles`:+ * another run will not fix them, a larger IX_PARSE_BUDGET_MS will.+ */+ parseTimeouts: string[];}-const SUPPORTED_NAMES = new Set([- ".gitignore", ".gitattributes", ".editorconfig", ".env",- ".eslintrc", ".prettierrc", ".babelrc",- "Makefile", "Dockerfile", "Procfile", "Gemfile", "Rakefile",- "BUILD", "WORKSPACE",-]);--const IGNORE_DIRS = new Set([- "node_modules", ".git", "dist", "build", "target", ".next",- ".cache", "__pycache__", ".ix", ".claude",-]);-/**- * Walk a directory and collect file paths with supported extensions.- * Bounded to prevent runaway on huge repos.+ * Most files `ix status` reads when the workspace is not a git work tree; the+ * cap the old status walk had. A walk has no bound of its own, and status runs+ * on every agent turn: from a home directory it would read the whole disk.+ * Files past the cap are not checked for changes. They do not look deleted:+ * a baseline entry counts as deleted only when the file is gone from disk.*/-function collectFiles(dir: string, limit: number = 5000): string[] {- const results: string[] = [];- const stack = [dir];+const STATUS_WALK_LIMIT = 5000;- while (stack.length > 0 && results.length < limit) {- const current = stack.pop()!;- let entries: fs.Dirent[];- try {- entries = fs.readdirSync(current, { withFileTypes: true });- } catch {- continue;- }- for (const entry of entries) {- if (results.length >= limit) break;- if (entry.name.startsWith(".") && entry.isDirectory()) continue;- if (IGNORE_DIRS.has(entry.name)) continue;-- const fullPath = path.join(current, entry.name);- if (entry.isDirectory()) {- stack.push(fullPath);- } else if (entry.isFile()) {- const ext = path.extname(entry.name).toLowerCase();- if (SUPPORTED_EXTENSIONS.has(ext) || SUPPORTED_NAMES.has(entry.name)) {- results.push(fullPath);- }- }- }- }- return results;+/**+ * The files `ix map` would discover (see `file-discovery.ts`), so "changed+ * since the last ingest" is judged over the same set the ingest recorded. A+ * git listing is complete; a walk stops at `walkLimit`.+ */+function collectFiles(dir: string, walkLimit: number): string[] {+ return discoverSourceFiles(dir, { walkLimit });}function differsFromIngestBaseline(@@ -77,6 +54,16 @@ function differsFromIngestBaseline(: ingestedMtime !== mtimeMs;}+/**+ * Absolute paths for a baseline's list of files the last run could not+ * ingest. They count as changed whatever their mtime: a new one is absent from+ * `files` with an mtime older than `lastIngestAt`, which would otherwise read+ * as current.+ */+function absoluteSet(workspaceRoot: string, relPaths: readonly string[]): Set<string> {+ return new Set(relPaths.map((rel) => path.resolve(workspaceRoot, rel)));+}+/*** Detect files that have been modified since the last ingest.* Uses the workspace-local ingest baseline so another workspace cannot advance it.@@ -91,7 +78,8 @@ function differsFromIngestBaseline(*/export function detectStaleFiles(root: string,- maxSamples: number = 5+ maxSamples: number = 5,+ walkLimit: number = STATUS_WALK_LIMIT,): StaleInfo {const workspaceRoot = path.resolve(root);const baseline = loadIngestBaseline(workspaceRoot);@@ -104,34 +92,64 @@ export function detectStaleFiles(staleFiles: 0,sampleChangedFiles: [],replayedFiles: [],+ parseTimeouts: [],};}- const files = collectFiles(workspaceRoot);- const currentFiles = new Set(files.map((filePath) => path.resolve(filePath)));+ // Discovery returns canonical paths (macOS `/var` is `/private/var`, a+ // Windows 8.3 name is expanded), while the baseline's keys sit under the root+ // as it was given. Compare and display both under the canonical root.+ const canonicalRoot = canonicalizeDiscoveredFilePath(workspaceRoot);+ const underCanonicalRoot = (absolutePath: string): string =>+ canonicalRoot !== workspaceRoot && absolutePath.startsWith(workspaceRoot + path.sep)+ ? canonicalRoot + absolutePath.slice(workspaceRoot.length)+ : absolutePath;+ const ingestedMtimes = new Map<string, number>();+ for (const [ingestedPath, mtime] of baseline.files) {+ const absolutePath = path.isAbsolute(ingestedPath)+ ? path.resolve(ingestedPath)+ : path.resolve(workspaceRoot, ingestedPath);+ ingestedMtimes.set(underCanonicalRoot(absolutePath), mtime);+ }++ const files = collectFiles(workspaceRoot, walkLimit).map(underCanonicalRoot);+ const currentFiles = new Set(files);const changedFiles: string[] = [];+ // Under the canonical root, like everything else compared here.+ const canonicalSet = (relPaths: readonly string[]): Set<string> =>+ new Set([...absoluteSet(workspaceRoot, relPaths)].map(underCanonicalRoot));+ const pending = canonicalSet(baseline.pendingFiles);+ // Canonical path -> the baseline's own workspace-relative spelling, which is+ // what status reports: POSIX separators on every platform, as the ingest+ // summary names them.+ const timedOut = new Map(+ baseline.parseTimeouts.map((rel) => [underCanonicalRoot(path.resolve(workspaceRoot, rel)), rel]),+ );+ const parseTimeouts: string[] = [];for (const filePath of files) {+ const timedOutAs = timedOut.get(filePath);+ if (timedOutAs !== undefined) {+ parseTimeouts.push(timedOutAs);+ continue;+ }try {const stat = fs.statSync(filePath);if (- differsFromIngestBaseline(filePath, stat.mtimeMs, baseline.files, baseline.lastIngestAt)+ pending.has(filePath)+ || differsFromIngestBaseline(filePath, stat.mtimeMs, ingestedMtimes, baseline.lastIngestAt)) {// Make path relative to root for display- const relative = path.relative(workspaceRoot, filePath);- changedFiles.push(relative);+ changedFiles.push(path.relative(canonicalRoot, filePath));}} catch {// skip inaccessible files}}- for (const ingestedPath of baseline.files.keys()) {- const absolutePath = path.isAbsolute(ingestedPath)- ? path.resolve(ingestedPath)- : path.resolve(workspaceRoot, ingestedPath);+ for (const absolutePath of ingestedMtimes.keys()) {if (!currentFiles.has(absolutePath) && !fs.existsSync(absolutePath)) {- changedFiles.push(path.relative(workspaceRoot, absolutePath));+ changedFiles.push(path.relative(canonicalRoot, absolutePath));}}@@ -143,6 +161,7 @@ export function detectStaleFiles(staleFiles: changedFiles.length,sampleChangedFiles: changedFiles.slice(0, maxSamples),replayedFiles: baseline.replayedFiles,+ parseTimeouts,};}@@ -196,6 +215,9 @@ export function isFileStale(filePath: string): boolean {export function createStaleProbe(): (filePath: string) => boolean {const workspaceRoot = path.resolve(resolveWorkspaceRoot());const baseline = loadIngestBaseline(workspaceRoot);+ const pending = baseline+ ? absoluteSet(workspaceRoot, [...baseline.pendingFiles, ...baseline.parseTimeouts])+ : new Set<string>();return (filePath: string): boolean => {// No baseline means the question this probe answers — "did this file change@@ -213,6 +235,7 @@ export function createStaleProbe(): (filePath: string) => boolean {if (!fs.existsSync(absolutePath)) {return baseline.files.has(absolutePath) || baseline.files.has(filePath);}+ if (pending.has(absolutePath)) return true;try {return differsFromIngestBaseline(diff --git a/ix-cli/test/perf/budgets.json b/ix-cli/test/perf/budgets.jsonindex 6e5dbc52..61a003d7 100644--- a/ix-cli/test/perf/budgets.json+++ b/ix-cli/test/perf/budgets.json@@ -8,7 +8,7 @@},"map (no-op)": {"requests": 6,- "bytes": 914,+ "bytes": 1017,"wallMs": 448},"map (one-line TS edit)": {diff --git a/skills/ix/references/flags.md b/skills/ix/references/flags.mdindex 5997c1c1..0b001339 100644--- a/skills/ix/references/flags.md+++ b/skills/ix/references/flags.md@@ -89,6 +89,13 @@ surfaces (#575).directory inside it refreshes just that part, leaving the rest of theworkspace alone. Only a path in no workspace and no repository becomes aworkspace of its own.+- **`ix ingest <path>` coalesces with a running `ix map` or `ix ingest` of the+ same workspace**, as `ix map` does: it does not run beside the holder, asks+ it for one more pass, and exits 0 (or `IX_MAP_COALESCE_EXIT_CODE`). With+ `--format json` it prints `{"coalesced": true, "workspace": <root>}`. The+ holder runs the extra pass before it exits: `ix map` re-ingests the+ workspace, and so does `ix ingest` once the workspace has a baseline+ (otherwise it re-ingests only its own path).- **`ix ingest` honours `--exclude <glob>` and a `.ixignore` at the workspaceroot**, matched relative to that root. A deliberate subset of `.gitignore`: `#` comments, `*`, `?`, `**`, aleading `/` to anchor at the root, a trailing `/` for directories only, and adiff --git a/skills/ix/references/troubleshooting.md b/skills/ix/references/troubleshooting.mdindex 0c0666d7..dba49c2d 100644--- a/skills/ix/references/troubleshooting.md+++ b/skills/ix/references/troubleshooting.md@@ -58,6 +58,15 @@ again.another writer (a second `ix map` on the same backend) moved the graphrevision under it (default 8). Only a commit that loses every retry iscounted as failed and left for the next run.+- `IX_PARSE_BUDGET_MS=N` — how long `ix map` / `ix ingest` may spend parsing+ one file (default 10000). A file that runs past it is skipped and named in+ the summary (`parseTimeouts` in `--format json`). It is not recorded as+ ingested: every later run tries it again, and `ix status` warns about it by+ name (`parseTimeouts` in `--format json`, `parse_timeouts` in `--format llm`)+ until a run gets it in. `0` turns the budget off.+- `IX_PARSE_WORKERS=N` — the most parse workers `ix map` / `ix ingest` starts+ (default 8, never more than the cores less one). Each loads every grammar,+ so a small edit starts one; raise it for a faster first map on a large machine.## Harness presence (hermetic reproduction)