Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): incremental adds, per-file baseline, parse budget, lazy pool, one discovery (#754 #775 #759 #763 #778)

Ian Hock1f3148d5d7b9
diff --git a/core-ingestion/src/__tests__/parseFile.perf.test.ts b/core-ingestion/src/__tests__/parseFile.perf.test.ts
new file mode 100644
index 00000000..f7798e4a
--- /dev/null
+++ b/core-ingestion/src/__tests__/parseFile.perf.test.ts
@@ -0,0 +1,92 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { describe, expect, it } from 'vitest';
+
+import { parseFile } from '../index.js';
+
+/**
+ * Parse time must grow with the file, not with its square.
+ *
+ * Every use of an imported JS/TS name is checked for shadowing, and each check
+ * used to rescan every enclosing scope in full: a test file with thousands of
+ * calls to one imported helper took minutes, and held a parse worker the whole
+ * time. These bounds are loose on purpose: the linear version takes well
+ * under a second here, the quadratic one minutes, so a bound of 10 s holds on
+ * a slow or busy CI runner and still fails the quadratic version. Each test
+ * gets its own timeout, because vitest's default 5 s is less than the bound.
+ */
+
+/** The wall-clock bound for one parse, and the per-test timeout around it. */
+const BOUND_MS = 10_000;
+const TEST_TIMEOUT_MS = 60_000;
+
+/** `n` calls to an imported function, all in one function body. */
+function importedCalls(n: number): string {
+ const lines = [`import { foo } from './x';`, `export function big() {`];
+ for (let i = 0; i < n; i++) lines.push(` foo(${i});`);
+ lines.push('}');
+ return lines.join('\n');
+}
+
+/** A vitest-shaped file: `n` top-level-ish `it` blocks inside one `describe`. */
+function vitestShaped(n: number): string {
+ const lines = [`import { describe, it, expect } from 'vitest';`, `import { g } from './g';`, `describe('suite', () => {`];
+ for (let i = 0; i < n; i++) {
+ lines.push(` it('case ${i}', () => {`, ` const r = g(${i});`, ` expect(r).toBe(${i});`, ` });`);
+ }
+ lines.push('});');
+ return lines.join('\n');
+}
+
+function timed(source: string): { ms: number; parsed: ReturnType<typeof parseFile> } {
+ const start = performance.now();
+ const parsed = parseFile('a/b.test.ts', source, { budgetMs: 0 });
+ return { ms: performance.now() - start, parsed };
+}
+
+describe('parseFile scales linearly with uses of an imported name', () => {
+ it('parses a 4,000-call file in seconds, not minutes', () => {
+ const { ms, parsed } = timed(importedCalls(4000));
+ expect(parsed).not.toBeNull();
+ expect(ms).toBeLessThan(BOUND_MS);
+ }, TEST_TIMEOUT_MS);
+
+ it('parses a 1,000-test vitest file in seconds, not minutes', () => {
+ const { ms, parsed } = timed(vitestShaped(1000));
+ expect(parsed).not.toBeNull();
+ expect(ms).toBeLessThan(BOUND_MS);
+ }, TEST_TIMEOUT_MS);
+
+ it('roughly quadruples in time when the file quadruples', () => {
+ timed(importedCalls(500)); // warm the parser and query cache
+ // Best of three each, so one scheduling hiccup cannot fail the ratio. A
+ // factor of 4 rather than 2 keeps linear (about 4) and quadratic (about 16)
+ // far apart, so load on a shared runner cannot push one across the bound.
+ const best = (n: number) => Math.min(...[0, 1, 2].map(() => timed(importedCalls(n)).ms));
+ const small = best(1000);
+ const large = best(4000);
+ expect(large / small).toBeLessThan(8);
+ }, TEST_TIMEOUT_MS);
+});
+
+describe('parseFile budget', () => {
+ it('gives up on a file that runs past its budget and says why', () => {
+ const failures: Array<{ reason: string }> = [];
+ const parsed = parseFile('a/b.test.ts', vitestShaped(2000), {
+ budgetMs: 1,
+ onFailure: (failure) => failures.push(failure),
+ });
+ expect(parsed).toBeNull();
+ expect(failures.map((f) => f.reason)).toEqual(['timeout']);
+ });
+
+ it('a budget of 0 is no budget', () => {
+ expect(parseFile('a/b.test.ts', vitestShaped(200), { budgetMs: 0 })).not.toBeNull();
+ });
+
+ it('a file no grammar handles is not a failure', () => {
+ const failures: unknown[] = [];
+ expect(parseFile('notes.unknownext', 'hello', { onFailure: (f) => failures.push(f) })).toBeNull();
+ expect(failures).toEqual([]);
+ });
+});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index 7ed4b5dc..d1527967 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -706,9 +706,12 @@ function jsTsDeclarationBinds(node: any, name: string, namespace: 'value' | 'typ
return false;
}
+// `namedChildren` once rather than `namedChild(i)` per index: each indexed
+// call walks the children from the start, so the loop was quadratic in the
+// scope's size before it did any work.
function jsTsScopeDirectlyBinds(scope: any, name: string, namespace: 'value' | 'type'): boolean {
- for (let i = 0; i < scope.namedChildCount; i++) {
- if (jsTsDeclarationBinds(scope.namedChild(i), name, namespace)) return true;
+ for (const child of scope.namedChildren) {
+ if (jsTsDeclarationBinds(child, name, namespace)) return true;
}
return false;
}
@@ -717,19 +720,50 @@ function jsTsFunctionHasVarBinding(node: any, name: string, isRoot = true): bool
if (!node) return false;
if (!isRoot && JS_TS_FUNCTION_NODES.has(node.type)) return false;
if (node.type === 'variable_declaration' && jsTsDeclarationBinds(node, name, 'value')) return true;
- for (let i = 0; i < node.namedChildCount; i++) {
- if (jsTsFunctionHasVarBinding(node.namedChild(i), name, false)) return true;
+ for (const child of node.namedChildren) {
+ if (jsTsFunctionHasVarBinding(child, name, false)) return true;
}
return false;
}
+/**
+ * Answers of the two whole-scope scans behind `isJsTsImportShadowed`, for one
+ * parse: node id -> `${kind}\0${namespace}\0${name}` -> result. Without it each
+ * USE of an imported name rescanned every enclosing scope, so a file with 4,000
+ * `it(...)` calls scanned its 4,000 top-level statements 4,000 times. Node ids
+ * are unique only within one tree, hence one memo per parse.
+ */
+type JsTsShadowMemo = Map<number, Map<string, boolean>>;
+
+function memoisedScan(memo: JsTsShadowMemo | undefined, node: any, key: string, scan: () => boolean): boolean {
+ if (!memo) return scan();
+ let forNode = memo.get(node.id);
+ if (!forNode) {
+ forNode = new Map();
+ memo.set(node.id, forNode);
+ }
+ let result = forNode.get(key);
+ if (result === undefined) {
+ result = scan();
+ forNode.set(key, result);
+ }
+ return result;
+}
+
/** True when a declaration shadows an imported JS/TS name at this use. */
-function isJsTsImportShadowed(identifierNode: any, name: string, namespace: 'value' | 'type'): boolean {
+function isJsTsImportShadowed(
+ identifierNode: any,
+ name: string,
+ namespace: 'value' | 'type',
+ memo?: JsTsShadowMemo,
+): boolean {
let child = identifierNode;
let scope = identifierNode?.parent;
while (scope) {
if (scope.type === 'statement_block' || scope.type === 'program') {
- if (jsTsScopeDirectlyBinds(scope, name, namespace)) return true;
+ if (memoisedScan(memo, scope, `scope\0${namespace}\0${name}`, () => jsTsScopeDirectlyBinds(scope, name, namespace))) {
+ return true;
+ }
}
if (namespace === 'value' && scope.type === 'catch_clause') {
if (jsTsBindingPatternContains(scope.childForFieldName?.('parameter'), name)) return true;
@@ -743,7 +777,7 @@ function isJsTsImportShadowed(identifierNode: any, name: string, namespace: 'val
if (scope.childForFieldName?.('name')?.text === name) return true;
if (jsTsBindingPatternContains(scope.childForFieldName?.('parameters'), name)) return true;
const body = scope.childForFieldName?.('body');
- if (body && jsTsFunctionHasVarBinding(body, name)) return true;
+ if (body && memoisedScan(memo, body, `var\0${name}`, () => jsTsFunctionHasVarBinding(body, name))) return true;
} else if (jsTsTypeParametersContain(scope.childForFieldName?.('type_parameters'), name)) {
return true;
}
@@ -2435,7 +2469,31 @@ function phpNamespaceForNode(node: any, spans: PhpNamespaceSpan[]): string | und
return active;
}
-export function parseFile(filePath: string, source: string): FileParseResult | null {
+/** Default per-file parse budget in ms, overridable with IX_PARSE_BUDGET_MS. */
+export const DEFAULT_PARSE_BUDGET_MS = 10_000;
+
+/** The per-file budget: IX_PARSE_BUDGET_MS if it is a non-negative number, else the default. 0 = none. */
+export function parseBudgetMs(): number {
+ const raw = process.env.IX_PARSE_BUDGET_MS;
+ const n = raw === undefined || raw.trim() === '' ? Number.NaN : Number(raw);
+ return Number.isFinite(n) && n >= 0 ? n : DEFAULT_PARSE_BUDGET_MS;
+}
+
+/** Thrown inside `parseFile` when a file runs past its budget; never escapes it. */
+class ParseBudgetExceeded extends Error {}
+
+export interface ParseFileOptions {
+ /** Give up on the file after this many ms. Defaults to `parseBudgetMs()`; 0 turns the budget off. */
+ budgetMs?: number;
+ /**
+ * Called when a supported file does not parse, before `parseFile` returns
+ * null: `timeout` when it ran past the budget, `error` for anything else.
+ * Not called for a file no grammar handles.
+ */
+ onFailure?: (failure: { reason: 'timeout' | 'error'; message: string }) => void;
+}
+
+export function parseFile(filePath: string, source: string, opts: ParseFileOptions = {}): FileParseResult | null {
const language = detectLanguageForSource(filePath, source);
if (!language) return null;
if (language === SupportedLanguages.YAML) return parseYamlFile(filePath, source);
@@ -2455,6 +2513,21 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const queries = LANGUAGE_QUERIES[language];
if (!queries) return null;
+ const budgetMs = opts.budgetMs ?? parseBudgetMs();
+ const deadline = budgetMs > 0 ? performance.now() + budgetMs : Number.POSITIVE_INFINITY;
+ let budgetTicks = 0;
+ /**
+ * Called once per match in both passes. The tree-sitter parse and query run
+ * in native code and cannot be interrupted, but the passes over the matches
+ * are where a file can go quadratic, and a file that does must not hold a
+ * parse worker for minutes. The worker pool has a backstop for the rest.
+ */
+ const checkBudget = (): void => {
+ if ((++budgetTicks & 63) === 0 && performance.now() > deadline) {
+ throw new ParseBudgetExceeded(`parse budget of ${budgetMs} ms exceeded`);
+ }
+ };
+
try {
const parser = getParser();
if (grammar !== _currentGrammar) {
@@ -2570,6 +2643,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
}
}
}
+ const jsTsShadowMemo: JsTsShadowMemo = new Map();
const recordJsTsImportUse = (
predicate: string,
srcName: string,
@@ -2580,7 +2654,10 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
): void => {
if (!isJsTs || !jsTsImportedLocalNames.has(localName)) return;
const key = `${predicate}\x00${srcName}\x00${dstName}`;
- const unshadowed = !isJsTsImportShadowed(identifierNode, localName, namespace);
+ // Every use is still evaluated: the edge is kept when ANY use is
+ // unshadowed, so the check cannot move behind the de-dup. The memo makes
+ // each use cost its scope depth rather than the size of its scopes.
+ const unshadowed = !isJsTsImportShadowed(identifierNode, localName, namespace, jsTsShadowMemo);
jsTsImportUseHasUnshadowed.set(key, (jsTsImportUseHasUnshadowed.get(key) ?? false) || unshadowed);
};
@@ -2604,6 +2681,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
// --- First pass: collect definitions ---
for (const match of pass1Matches) {
+ checkBudget();
// Definition captures: name + definition.*
const defCapture = match.captures.find((c: any) =>
c.name.startsWith('definition.')
@@ -2879,6 +2957,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
// --- Second pass: calls and imports ---
for (const match of pass2Matches) {
+ checkBudget();
// Full import statement captures (Scala: reconstructs dotted package paths)
const importStmt = match.captures.find((c: any) => c.name === 'import.stmt');
if (importStmt) {
@@ -3381,6 +3460,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
};
} catch (e) {
if (process.env.IX_PARSE_DEBUG === '1') console.error('parseFile threw:', e);
+ opts.onFailure?.({ reason: e instanceof ParseBudgetExceeded ? 'timeout' : 'error', message: String(e) });
return null;
}
}
diff --git a/core-ingestion/src/parse-worker.ts b/core-ingestion/src/parse-worker.ts
index bacbdbfb..7a0b925d 100644
--- a/core-ingestion/src/parse-worker.ts
+++ b/core-ingestion/src/parse-worker.ts
@@ -5,7 +5,9 @@
* Each worker maintains its own Parser singleton (safe — module state is
* per-thread).
* Receives: { filePath: string, source: string } | { __shutdown: true }
- * Posts: { ok: true, result: FileParseResult } | { ok: false }
+ * Posts: { ok: true, result: FileParseResult }
+ * | { ok: false, result: null, reason?: 'timeout' | 'error', message?: string }
+ * (no reason: no grammar handles the file)
*/
import { parentPort } from 'node:worker_threads';
import { parseFile } from './index.js';
@@ -88,10 +90,13 @@ parentPort.on('message', (msg: ParseMessage | ShutdownMessage) => {
return;
}
const { filePath, source } = msg as ParseMessage;
+ // Why a file came back null, so the pool can name the ones that ran past
+ // the parse budget instead of folding them into "unparsed".
+ let failure: { reason: 'timeout' | 'error'; message: string } | undefined;
try {
- const result = parseFile(filePath, source);
- parentPort!.postMessage({ ok: result !== null, result: result ?? null });
- } catch {
- parentPort!.postMessage({ ok: false, result: null });
+ const result = parseFile(filePath, source, { onFailure: (f) => { failure = f; } });
+ parentPort!.postMessage({ ok: result !== null, result: result ?? null, ...(failure ?? {}) });
+ } catch (err) {
+ parentPort!.postMessage({ ok: false, result: null, reason: 'error', message: String(err) });
}
});
diff --git a/docs-site/src/content/docs/reference/commands.md b/docs-site/src/content/docs/reference/commands.md
index 612e2913..b309b5ad 100644
--- a/docs-site/src/content/docs/reference/commands.md
+++ b/docs-site/src/content/docs/reference/commands.md
@@ -74,6 +74,11 @@ workspace is left alone. A path outside every workspace and repository is ingest
`.gitignore` syntax: `#` comments, `*`, `?`, `**`, a leading `/` to anchor to the root, a trailing `/` for
directories, and bare names that match at any depth. There is no `!` negation.
+`ix ingest <path>` and `ix map` take the same per-workspace lock. A run that finds it held does not run beside
+the holder: it asks the holder for one more pass, prints a note (or `{"coalesced": true, ...}` with
+`--format json`) and exits 0, or with `IX_MAP_COALESCE_EXIT_CODE` if that is set. The holder runs that pass
+before it exits: `ix map` re-ingests the workspace, and so does `ix ingest` when the workspace has been mapped.
+
## Setup and health
| Command | Does |
diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.md
index 0dde0ceb..e0f922a9 100644
--- a/docs-site/src/content/docs/reference/configuration.md
+++ b/docs-site/src/content/docs/reference/configuration.md
@@ -23,6 +23,8 @@ ix config set format llm # set a value
| `IX_MCP_SUBPROCESS=1` | Runs each MCP tool call in its own `ix` process |
| `IX_COMMIT_FAILURE_LIMIT=<n>` | Consecutive failed commits before `ix map` stops (default 5). `0` never stops |
| `IX_COMMIT_BASE_REV_RETRIES=<n>` | Re-sends of a commit after another writer on the same backend moved the graph revision under it (default 8) |
+| `IX_PARSE_BUDGET_MS=<ms>` | How long `ix map` / `ix ingest` may spend parsing one file before skipping it and naming it in the summary (default 10000). A skipped file is tried again on every run, and `ix status` warns about it until it is in the graph. `0` turns the budget off |
+| `IX_PARSE_WORKERS=<n>` | Most parse workers `ix map` / `ix ingest` starts (default 8, never more than the cores less one). Workers start as files need them, one per 50 files |
### Bootstrap script variables
diff --git a/docs/api/README.md b/docs/api/README.md
index d429efbb..47d22838 100644
--- a/docs/api/README.md
+++ b/docs/api/README.md
@@ -167,7 +167,14 @@ Ingest a path into the graph. Long-running: the client allows **30 minutes**.
> under the same key: `unchanged` there counts only files skipped as *mtime- or
> hash-unchanged*, `emptyFile` is a real count rather than a hardcoded `0`, and
> there is an extra `unparsed` bucket for files the parse pool returned nothing
-> for. That is the client's own summary of its own run and is not this response;
+> for. A file whose parse ran past `IX_PARSE_BUDGET_MS` is counted in
+> `parseTimeout` and named in a top-level `parseTimeouts` list. Two buckets
+> come from discovery and are not part of `filesSkipped`: `skippedDirs` counts
+> files left out under a directory that is never source (build output,
+> dependencies, VCS metadata, tool caches) -- by name on a plain walk, and only
+> for untracked files in a git work tree, where a tracked file is always kept
+> -- and `unreadable` counts listed files that could not be stat'd.
+> That is the client's own summary of its own run and is not this response;
> see the stitch section below for why the narrower `unchanged` is load-bearing
> there.
diff --git a/ix-cli/src/cli/__tests__/file-discovery.test.ts b/ix-cli/src/cli/__tests__/file-discovery.test.ts
new file mode 100644
index 00000000..496a6ada
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/file-discovery.test.ts
@@ -0,0 +1,123 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { execFileSync } from 'node:child_process';
+import { cpSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, relative } from 'node:path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { discoverSourceFiles, emptyDiscoveryCounts, tryGitLsFiles, walkFiles } from '../file-discovery.js';
+
+const scratch: string[] = [];
+afterEach(() => {
+ for (const dir of scratch.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function tree(files: Record<string, string>): string {
+ const root = realpathSync.native(mkdtempSync(join(tmpdir(), 'ix-files-')));
+ scratch.push(root);
+ for (const [rel, text] of Object.entries(files)) {
+ mkdirSync(dirname(join(root, rel)), { recursive: true });
+ writeFileSync(join(root, rel), text, 'utf8');
+ }
+ return root;
+}
+
+const git = (root: string, ...args: string[]) => execFileSync('git', args, { cwd: root, stdio: 'ignore' });
+const rel = (root: string, files: string[]) => files.map(f => relative(root, f).replace(/\\/g, '/')).sort();
+
+describe('file discovery', () => {
+ it('keeps non-ASCII file names in a git repository (they used to be quoted and dropped)', () => {
+ const root = tree({ 'src/café.ts': 'export const a = 1;\n', 'src/日本語.ts': 'export const b = 2;\n', 'src/plain.ts': '' });
+ git(root, 'init', '-q');
+ git(root, 'add', '-A');
+ expect(rel(root, tryGitLsFiles(root, true)!)).toEqual(['src/café.ts', 'src/plain.ts', 'src/日本語.ts']);
+ });
+
+ it('git and the walk return the same files for the same tree', () => {
+ const files = {
+ '.gitignore': 'generated/\nlocal.ts\n',
+ 'src/app.ts': '',
+ 'src/local.ts': '',
+ 'src/generated/out.ts': '',
+ 'tests/test_app.py': '',
+ 'pkg/server/server.go': '',
+ 'examples/demo.ts': '',
+ '.github/workflows/ci.yml': 'on: push\n',
+ 'node_modules/dep/index.js': '',
+ 'dist/bundle.js': '',
+ 'pkg/api/types.pb.go': '',
+ };
+ const gitRoot = tree(files);
+ git(gitRoot, 'init', '-q');
+ const walkRoot = tree(files); // no .git: the walk
+
+ const viaGit = rel(gitRoot, discoverSourceFiles(gitRoot));
+ const viaWalk = rel(walkRoot, discoverSourceFiles(walkRoot));
+
+ expect(viaWalk).toEqual(viaGit);
+ expect(viaWalk).toEqual(['.github/workflows/ci.yml', 'examples/demo.ts', 'pkg/server/server.go', 'src/app.ts', 'tests/test_app.py']);
+ });
+
+ it('honours a nested .gitignore on the walk', () => {
+ const root = tree({ 'a/.gitignore': 'secret.ts\n', 'a/secret.ts': '', 'a/kept.ts': '', 'b/secret.ts': '' });
+ expect(rel(root, [...walkFiles(root, true)])).toEqual(['a/kept.ts', 'b/secret.ts']);
+ });
+
+ it('counts what it skipped instead of dropping it silently', () => {
+ const root = tree({ 'src/a.ts': '', 'src/gone.ts': '' });
+ git(root, 'init', '-q');
+ git(root, 'add', '-A');
+ rmSync(join(root, 'src', 'gone.ts')); // tracked, no longer on disk
+ writeFileSync(join(root, 'src', 'b.js'), '', 'utf8');
+ mkdirSync(join(root, 'dist'));
+ writeFileSync(join(root, 'dist', 'b.js'), '', 'utf8'); // untracked build output
+ const counts = emptyDiscoveryCounts();
+ expect(rel(root, discoverSourceFiles(root, { counts }))).toEqual(['src/a.ts', 'src/b.js']);
+ expect(counts).toEqual({ skippedDirs: 1, unreadable: 1 });
+ });
+
+ it('keeps a tracked file whatever its directory is called; skips untracked ones under those names', () => {
+ // A repository that commits `build/` or `out/` is saying it is source.
+ // git's ignore rules decide what is generated in a work tree; the names
+ // are only a fallback for what nothing told git to ignore.
+ const files = {
+ 'build/gen.ts': '', 'out/main.ts': '', 'target/T.java': '', 'obj/x.cs': '', 'coverage/report.js': '',
+ 'src/app.ts': '',
+ };
+ const root = tree(files);
+ git(root, 'init', '-q');
+ git(root, 'add', '-A');
+ writeFileSync(join(root, 'build', 'fresh.ts'), '', 'utf8'); // untracked, not ignored
+ const counts = emptyDiscoveryCounts();
+ expect(rel(root, discoverSourceFiles(root, { counts }))).toEqual([
+ 'build/gen.ts', 'coverage/report.js', 'obj/x.cs', 'out/main.ts', 'src/app.ts', 'target/T.java',
+ ]);
+ expect(counts.skippedDirs).toBe(1);
+
+ // Without git there is no record of what is tracked: the walk keeps
+ // skipping those names.
+ const plain = tree(files);
+ expect(rel(plain, discoverSourceFiles(plain))).toEqual(['src/app.ts']);
+ });
+
+ it('stops a walk at walkLimit, and never caps a git listing', () => {
+ const root = tree({ 'a.ts': '', 'b.ts': '', 'c.ts': '', 'd.ts': '' });
+ expect(discoverSourceFiles(root, { walkLimit: 2 })).toHaveLength(2);
+ git(root, 'init', '-q');
+ expect(discoverSourceFiles(root, { walkLimit: 2 })).toHaveLength(4);
+ });
+
+ it('copies of one tree agree whether or not they are a git checkout', () => {
+ // The same tree as a git repository with committed files and as a plain
+ // copy: discovery must not depend on how it was obtained.
+ const root = tree({ 'lib/x.ts': '', 'lib/tests/x.test.ts': '', 'scripts/tool.py': '' });
+ git(root, 'init', '-q');
+ git(root, 'add', '-A');
+ const copy = realpathSync.native(mkdtempSync(join(tmpdir(), 'ix-files-copy-')));
+ scratch.push(copy);
+ cpSync(root, copy, { recursive: true, filter: (src) => !src.includes(`${'/'}.git`) });
+ expect(rel(copy, discoverSourceFiles(copy))).toEqual(rel(root, discoverSourceFiles(root)));
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
index 7dedc8e4..de47d427 100644
--- a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
@@ -165,3 +165,16 @@ describe("extractor re-ingest progress", () => {
expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);
});
});
+
+describe("ingest baseline atomic write", () => {
+ it("leaves no temp file behind when the rename fails", () => {
+ const root = path.join(home, "p");
+ const target = ingestMtimeCachePath(root);
+ // A directory where the baseline goes makes the rename fail.
+ fs.mkdirSync(target, { recursive: true });
+
+ saveIngestBaseline(root, new Map([["a.ts", 1_000]]), 3);
+
+ expect(fs.readdirSync(path.dirname(target)).filter(name => name.endsWith(".tmp"))).toEqual([]);
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
index 9c53eaef..3b207b3e 100644
--- a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
@@ -241,9 +241,9 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {
await runSequence("head", [edit("editBody", "pkg/app.py"), edit("editBody", "src/com/ex/App.java")]);
});
- // F-07. A new file has no backend hash, so the whole repository takes the
- // first-ingest path and every file is parsed and sent again.
- it.fails("IN-01: adding a file sends one patch", async () => {
+ // F-07. A new file has no backend hash; before IN-01 that sent the whole
+ // repository down the first-ingest path, every file parsed and sent again.
+ it("IN-01: adding a file sends one patch", async () => {
await runSequence("head", [edit("addFile", undefined, 0)], {
langs: ["ts"],
check: (world) => {
@@ -255,6 +255,14 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {
});
});
+ // The index prescan holds no Python or Java symbols, so an added file in
+ // either language still takes the whole-repository pass: resolved alone, its
+ // cross-file calls would point at a module node or at nothing.
+ it("IN-01: an added Python or Java file keeps its cross-file calls", async () => {
+ await runSequence("head", [edit("addFile", undefined, 0)], { langs: ["py"] });
+ await runSequence("head", [edit("addFile", undefined, 1)], { langs: ["java"] });
+ });
+
// F-04. Dropping the file's last import removes a node, the patch carries a
// DeleteNode, and a delete-bearing patch goes to `/v1/patch`, which does not
// sweep: the removed CALLS and IMPORTS edges stay live.
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 28b6bd1a..831d103d 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -6,7 +6,8 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { execFileSync } from "node:child_process";
-import { ingestFiles } from "../commands/ingest.js";
+import { ingestFiles, ingestPathSingleFlight } from "../commands/ingest.js";
+import { acquireMapLock, takeMapRerun } from "../single-flight.js";
import { ingestMtimeCachePath, ingestRebuildPath, loadConfig } from "../config.js";
import { workspaceIdForPath } from "../system.js";
import { FakeBackend } from "./helpers/fake-backend.js";
@@ -679,6 +680,121 @@ describe("ingestFiles against a fake backend", () => {
expect(baselineFiles()).toEqual(expect.arrayContaining([join(repo, "src", "m000.ts"), join(repo, "src", "m002.ts")]));
});
+ it("names a file whose parse runs past the budget, ingests the rest, and retries it until it lands", async () => {
+ // About 2.5 s of parsing, under the 1 MB cap, against a 1 s budget.
+ // Whichever gives up first -- the worker's own check or the pool's
+ // backstop at twice the budget --
+ // the file must be named, not folded silently into "unparsed".
+ fixture(3);
+ const calls = Array.from({ length: 60_000 }, (_, i) => ` foo(${i});`);
+ writeFileSync(join(repo, "src", "huge.ts"), [`import { foo } from './m000';`, "export function big() {", ...calls, "}"].join("\n"), "utf8");
+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });
+ const saved = process.env.IX_PARSE_BUDGET_MS;
+ process.env.IX_PARSE_BUDGET_MS = "1000";
+ const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ try {
+ const summary = await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+
+ expect(summary.parseTimeouts).toEqual(["src/huge.ts"]);
+ expect(backend.acceptedPatches(), "the other three still land").toBe(3);
+ expect(stderr.mock.calls.map(([c]) => String(c)).join("")).toContain("src/huge.ts");
+
+ // Skipped, not settled: it used to be recorded clean, so every later run
+ // skipped it as unchanged while status called the graph current.
+ const huge = join(repo, "src", "huge.ts");
+ const stored = loadIngestBaseline(repo)!;
+ expect(stored.files.has(huge), "not recorded as ingested").toBe(false);
+ expect(stored.parseTimeouts).toEqual(["src/huge.ts"]);
+ expect(stored.pendingFiles).toEqual([]);
+ const status = detectStaleFiles(repo);
+ expect(status.parseTimeouts).toEqual(["src/huge.ts"]);
+ expect(status.staleFiles, "named once, as a timeout").toBe(0);
+ const llm = renderStatusLlm("ok", "x", status);
+ expect(llm[0]).toContain("parse_timeouts=1");
+ expect(llm[0]).toContain("stale=true");
+
+ // With the budget off, the next run parses it and the warning goes.
+ process.env.IX_PARSE_BUDGET_MS = "0";
+ expect(backend.sourceUris).not.toContain("src/huge.ts");
+ const next = await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ expect(next.parseTimeouts).toEqual([]);
+ expect(backend.sourceUris, "retried, not skipped as unchanged").toContain("src/huge.ts");
+ expect(loadIngestBaseline(repo)!.files.get(huge)).toBe(statSync(huge).mtimeMs);
+ expect(loadIngestBaseline(repo)!.parseTimeouts).toEqual([]);
+ expect(detectStaleFiles(repo).parseTimeouts).toEqual([]);
+ } finally {
+ stderr.mockRestore();
+ if (saved === undefined) delete process.env.IX_PARSE_BUDGET_MS;
+ else process.env.IX_PARSE_BUDGET_MS = saved;
+ }
+ });
+
+ describe("a file the backend has never seen is an incremental change", () => {
+ // A new file has no source hash on the backend, so `knownHashes` came back
+ // empty and the whole repository took the first-ingest path: every file
+ // re-read, re-parsed and re-sent to add one.
+ const incremental = () =>
+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ const stage = () => execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });
+
+ beforeEach(() => {
+ fixture(16);
+ backend.rememberHashes = true;
+ });
+
+ it("adding one file sends one patch", async () => {
+ await incremental();
+ writeFileSync(join(repo, "src", "added.ts"), "export const added = 1;\n", "utf8");
+ stage();
+ backend.resetRequests();
+
+ const summary = await incremental();
+
+ expect(backend.acceptedPatches(), "patches on the wire").toBe(1);
+ expect(backend.commitCount).toBe(1);
+ expect(summary.filesSkippedAsUnchanged).toBe(16);
+ expect(summary.filesDiscovered).toBe(17);
+ });
+
+ it("an empty file that gains content sends one patch", async () => {
+ writeFileSync(join(repo, "src", "empty.ts"), "", "utf8");
+ stage();
+ await incremental();
+ writeFileSync(join(repo, "src", "empty.ts"), "export const filled = 1;\n", "utf8");
+ backend.resetRequests();
+
+ const summary = await incremental();
+
+ expect(backend.acceptedPatches()).toBe(1);
+ expect(summary.filesSkippedAsUnchanged).toBe(16);
+ });
+
+ it("a file that shrinks under the size cap sends one patch", async () => {
+ writeFileSync(join(repo, "src", "big.ts"), `export const big = "${"x".repeat(1024 * 1024)}";\n`, "utf8");
+ stage();
+ await incremental();
+ writeFileSync(join(repo, "src", "big.ts"), "export const big = 'small';\n", "utf8");
+ backend.resetRequests();
+
+ const summary = await incremental();
+
+ expect(backend.acceptedPatches()).toBe(1);
+ expect(summary.filesSkippedAsUnchanged).toBe(16);
+ });
+
+ it("a wiped backend still re-sends everything", async () => {
+ await incremental();
+ writeFileSync(join(repo, "src", "added.ts"), "export const added = 1;\n", "utf8");
+ stage();
+ backend.rememberHashes = false;
+ backend.resetRequests();
+
+ await incremental();
+
+ expect(backend.acceptedPatches(), "the DB-reset guard empties the baseline").toBe(17);
+ });
+ });
+
describe("a changed file the backend answers Idempotent is reported, not marked clean (F-01)", () => {
// A revert sends the patch id the backend committed for the original
// bytes. The shipped backend replays any id it has seen, so it answers
@@ -782,6 +898,190 @@ describe("ingestFiles against a fake backend", () => {
});
});
+ describe("one failing file does not stop the baseline", () => {
+ // The baseline used to be written only by a clean run. One unreadable file
+ // then meant no run ever recorded anything again: every map re-read the
+ // whole workspace, and deleted files were never retracted.
+ const incremental = () =>
+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ const stage = () => execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });
+ const stored = () => loadIngestBaseline(repo)!;
+ let stderr: ReturnType<typeof vi.spyOn>;
+ beforeEach(() => {
+ stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ });
+ afterEach(() => {
+ stderr.mockRestore();
+ process.exitCode = undefined;
+ });
+
+ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "records every other file and retracts a deletion while one file is unreadable",
+ async () => {
+ fixture(3);
+ // The graph model: a deletion reconciles through /v1/patches/:id and
+ // /v1/entity/:id, which only it serves.
+ backend.semantics = "head";
+ const broken = join(repo, "src", "m000.ts");
+ const deleted = join(repo, "src", "m001.ts");
+ await incremental();
+ const before = stored().files.get(broken);
+
+ writeFileSync(broken, "export const changed = 1;\n", "utf8");
+ rmSync(deleted);
+ stage(); // before the chmod: git cannot read the file either
+ execFileSync("chmod", ["000", broken]);
+ backend.resetRequests();
+ try {
+ const summary = await incremental();
+ expect(summary.parseErrors, "the unreadable file is an error").toBeGreaterThan(0);
+ } finally {
+ execFileSync("chmod", ["644", broken]);
+ }
+
+ const files = stored().files;
+ expect(files.has(deleted), "the deletion was retracted and recorded").toBe(false);
+ expect(backend.sourceUris).toContain("src/m001.ts");
+ expect(files.get(join(repo, "src", "m002.ts")), "a settled file is recorded").toBeDefined();
+ expect(files.get(broken), "the failed file keeps its old mtime").toBe(before);
+
+ // Readable again: the next run sends it, and only it.
+ backend.resetRequests();
+ await incremental();
+ expect(backend.acceptedPatches()).toBe(1);
+ expect(stored().files.get(broken)).toBe(statSync(broken).mtimeMs);
+ },
+ );
+
+ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
+ "a new file that fails is pending: retried, and ix status does not call the graph current",
+ async () => {
+ // A new file has no previous mtime to keep, so it dropped out of the
+ // baseline while `lastIngestAt` moved past its mtime, and status
+ // judged it current.
+ fixture(3);
+ await incremental();
+ const added = join(repo, "src", "added.ts");
+ writeFileSync(added, "export const added = 1;\n", "utf8");
+ stage();
+ execFileSync("chmod", ["000", added]);
+ try {
+ await incremental();
+ } finally {
+ execFileSync("chmod", ["644", added]);
+ }
+
+ expect(stored().files.has(added)).toBe(false);
+ expect(stored().pendingFiles).toEqual(["src/added.ts"]);
+ const status = detectStaleFiles(repo);
+ expect(status.staleFiles).toBe(1);
+ expect(status.sampleChangedFiles).toEqual(["src/added.ts"]);
+ expect(renderStatusLlm("ok", "x", status)[0]).toContain("stale=true");
+
+ // The next run sends it and clears the list.
+ await incremental();
+ expect(stored().files.get(added)).toBe(statSync(added).mtimeMs);
+ expect(stored().pendingFiles).toEqual([]);
+ expect(detectStaleFiles(repo).staleFiles).toBe(0);
+ },
+ );
+
+ it("a failed lookup is not a reset", async () => {
+ fixture(3);
+ backend.rememberHashes = true;
+ await incremental();
+ backend.failSourceHashes = true;
+ backend.resetRequests();
+
+ const summary = await incremental();
+
+ expect(summary.filesSkippedAsUnchanged, "still incremental").toBe(3);
+ expect(backend.commitCount).toBe(0);
+ });
+
+ it("leaves no temp file beside the baseline", async () => {
+ fixture(2);
+ await incremental();
+ const dir = join(home, ".ix");
+ expect(existsSync(ingestMtimeCachePath(repo))).toBe(true);
+ expect(execFileSync("ls", ["-a", dir], { encoding: "utf8" })).not.toMatch(/\.tmp/);
+ });
+ });
+
+ describe("ix ingest coalesces with a map or ingest of the same workspace", () => {
+ // It used to exit 1 with "run this again", which an editor hook or an MCP
+ // client that ingests after every edit reported as a failed ingest, while
+ // `ix map` in the same spot coalesces and exits 0.
+ let stdout: ReturnType<typeof vi.spyOn>;
+ let stderr: ReturnType<typeof vi.spyOn>;
+ beforeEach(() => {
+ stdout = vi.spyOn(console, "log").mockImplementation(() => undefined);
+ stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ process.exitCode = undefined;
+ });
+ afterEach(() => {
+ stdout.mockRestore();
+ stderr.mockRestore();
+ process.exitCode = undefined;
+ delete process.env.IX_MAP_COALESCE_EXIT_CODE;
+ });
+ const written = (spy: ReturnType<typeof vi.spyOn>) =>
+ spy.mock.calls.map((call: unknown[]) => String(call[0])).join("");
+
+ it("exits 0, asks the holder for a rerun, and prints JSON a client can parse", async () => {
+ fixture(1);
+ const held = acquireMapLock(repo, "test");
+ try {
+ expect(held).not.toBeNull();
+ backend.resetRequests();
+ await ingestPathSingleFlight(join(repo, "src"), { format: "json" });
+
+ expect(process.exitCode, "a coalesce is not a failure").toBeUndefined();
+ expect(backend.commitCount, "nothing ran beside the holder").toBe(0);
+ expect(JSON.parse(written(stdout))).toEqual({ coalesced: true, workspace: repo });
+ expect(takeMapRerun(repo), "the holder is asked to run once more").toBe(true);
+ } finally {
+ held?.release();
+ }
+ });
+
+ it("honours IX_MAP_COALESCE_EXIT_CODE, as ix map does", async () => {
+ fixture(1);
+ const held = acquireMapLock(repo, "test");
+ try {
+ process.env.IX_MAP_COALESCE_EXIT_CODE = "75";
+ await ingestPathSingleFlight(repo, { format: "text" });
+ expect(process.exitCode).toBe(75);
+ expect(written(stderr)).toContain("Another ix map or ix ingest is running");
+ } finally {
+ held?.release();
+ }
+ });
+
+ it("a holding ingest re-runs for the one that coalesced, so its edit lands", async () => {
+ fixture(3);
+ await ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ const first = join(repo, "src", "m000.ts");
+ const second = join(repo, "src", "m002.ts");
+ writeFileSync(first, "export const first = 1;\n", "utf8");
+ writeFileSync(second, "export const second = 2;\n", "utf8");
+
+ // The lock is taken before the first await, so the second call finds it
+ // held whatever the scheduling.
+ const holder = ingestPathSingleFlight(first, { format: "json" });
+ const waiter = ingestPathSingleFlight(second, { format: "json" });
+ await Promise.all([holder, waiter]);
+
+ expect(process.exitCode).toBeUndefined();
+ const files = loadIngestBaseline(repo)!.files;
+ expect(files.get(second), "the coalesced run's file was ingested").toBe(statSync(second).mtimeMs);
+ expect(files.get(first)).toBe(statSync(first).mtimeMs);
+ const after = acquireMapLock(repo, "after");
+ expect(after, "the lock was let go").not.toBeNull();
+ after?.release();
+ });
+ });
+
it("ingests only the languages --lang names, and keeps the rest of the baseline", async () => {
fixture(3);
writeFileSync(join(repo, "src", "tool.py"), "def tool():\n return 1\n", "utf8");
diff --git a/ix-cli/src/cli/__tests__/parse-pool-size.test.ts b/ix-cli/src/cli/__tests__/parse-pool-size.test.ts
new file mode 100644
index 00000000..051ee9b9
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/parse-pool-size.test.ts
@@ -0,0 +1,32 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { describe, expect, it } from "vitest";
+
+import { parsePoolSize } from "../commands/ingest.js";
+
+describe("parsePoolSize", () => {
+ it("gives a one-file edit one worker, however many cores there are", () => {
+ expect(parsePoolSize(1, 24, undefined)).toBe(1);
+ expect(parsePoolSize(50, 24, undefined)).toBe(1);
+ expect(parsePoolSize(51, 24, undefined)).toBe(2);
+ });
+
+ it("caps a large run at 8 by default, or at IX_PARSE_WORKERS", () => {
+ expect(parsePoolSize(10_000, 24, undefined)).toBe(8);
+ expect(parsePoolSize(10_000, 24, "16")).toBe(16);
+ expect(parsePoolSize(10_000, 24, "2")).toBe(2);
+ });
+
+ it("never exceeds the cores less one, and never drops below one", () => {
+ expect(parsePoolSize(10_000, 4, "16")).toBe(3);
+ expect(parsePoolSize(10_000, 2, undefined)).toBe(1);
+ expect(parsePoolSize(10_000, 1, undefined)).toBe(1);
+ expect(parsePoolSize(0, 24, undefined)).toBe(1);
+ });
+
+ it("ignores an IX_PARSE_WORKERS that is not a positive integer", () => {
+ for (const bad of ["0", "-3", "1.5", "lots", ""]) {
+ expect(parsePoolSize(10_000, 24, bad), bad).toBe(8);
+ }
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/parse-pool.test.ts b/ix-cli/src/cli/__tests__/parse-pool.test.ts
index 1fcbb3dd..8d479e1c 100644
--- a/ix-cli/src/cli/__tests__/parse-pool.test.ts
+++ b/ix-cli/src/cli/__tests__/parse-pool.test.ts
@@ -868,4 +868,109 @@ describe("ParsePool", () => {
expect(elapsed).toBeLessThan(3000);
expect(pool.crashedTasks(), "a clean teardown is not a crash").toBe(0);
}, 20000);
+
+ /**
+ * Blocks its event loop for 1.5 s on `slow.ts` -- standing in for a parse
+ * stuck in native code, which no budget check inside the worker can reach --
+ * then answers. Every other file answers at once.
+ */
+ const SLOW_ON_ONE = `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ if (msg.filePath === 'slow.ts') Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 6000);
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `;
+
+ it("settles a parse that holds its worker past the backstop as a timeout, and carries on", async () => {
+ // The backstop also runs for `a.ts` from the moment it is handed to the
+ // replacement worker, whose start-up counts against it. 1 s leaves room for
+ // that start on a busy runner; the slow parse holds its worker for 6 s.
+ const pool = new ParsePool(worker("slow", SLOW_ON_ONE), 1, undefined, undefined, 1000);
+ pool.init();
+ const start = Date.now();
+
+ const [slow, next] = await Promise.all([pool.parse("slow.ts", "x"), pool.parse("a.ts", "y")]);
+
+ expect(slow, "settled, not left pending").toBeNull();
+ expect(next, "a replacement worker took the queue").toEqual({ filePath: "a.ts" });
+ expect(Date.now() - start, "did not wait for the slow parse").toBeLessThan(5000);
+ expect(pool.timedOutFiles()).toEqual(["slow.ts"]);
+ expect(pool.crashedTasks(), "a timeout is not a crash: it recurs every run").toBe(0);
+ await pool.destroy();
+ });
+
+ /** Reports a budget timeout for `t.ts`, as the real worker does. */
+ const REPORTS_TIMEOUT = `
+ import { parentPort } from 'node:worker_threads';
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ if (msg.filePath === 't.ts') {
+ parentPort.postMessage({ ok: false, result: null, reason: 'timeout', message: 'budget' });
+ } else {
+ parentPort.postMessage({ ok: false, result: null, reason: 'error', message: 'boom' });
+ }
+ });
+ `;
+
+ /** Echoes, and appends a line to `marker` when it starts: one line per worker. */
+ const COUNTS_STARTS = (marker: string) => `
+ import { parentPort, threadId } from 'node:worker_threads';
+ import { appendFileSync } from 'node:fs';
+ appendFileSync(${JSON.stringify(marker)}, threadId + '\\n');
+ parentPort.on('message', (msg) => {
+ if (msg && msg.__shutdown) { parentPort.close(); return; }
+ parentPort.postMessage({ ok: true, result: { filePath: msg.filePath } });
+ });
+ `;
+
+ it("starts workers only as work arrives, up to its size", async () => {
+ // A one-file edit used to start a worker per core to parse one file.
+ const marker = join(dir, "starts");
+ const pool = new ParsePool(worker("counts", COUNTS_STARTS(marker)), 4);
+ const started = () => (existsSync(marker) ? readFileSync(marker, "utf8").trim().split("\n").length : 0);
+
+ await pool.parse("a.ts", "x");
+ expect(started(), "one task, one worker").toBe(1);
+
+ await Promise.all(["b", "c", "d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));
+ expect(started(), "never more than the pool's size").toBeLessThanOrEqual(4);
+
+ await pool.destroy();
+ });
+
+ it("grows when a later phase asks for more, and never shrinks", async () => {
+ const marker = join(dir, "grows");
+ const pool = new ParsePool(worker("grows", COUNTS_STARTS(marker)), 1);
+ const started = () => readFileSync(marker, "utf8").trim().split("\n").length;
+
+ await Promise.all(["a", "b", "c"].map((f) => pool.parse(`${f}.ts`, "x")));
+ expect(started()).toBe(1);
+
+ pool.growTo(3);
+ pool.growTo(2);
+ await Promise.all(["d", "e", "f", "g"].map((f) => pool.parse(`${f}.ts`, "x")));
+ expect(started()).toBeGreaterThan(1);
+ expect(started()).toBeLessThanOrEqual(3);
+
+ await pool.destroy();
+ });
+
+ it("names a file the worker reports as timed out, and only one that counts", async () => {
+ const pool = new ParsePool(worker("reports", REPORTS_TIMEOUT), 1);
+ pool.init();
+
+ const results = await Promise.all([
+ pool.parse("t.ts", "x"),
+ pool.parse("e.ts", "x"),
+ // The index prescan does not count its losses; see `parse`.
+ pool.parse("t.ts", "x", false),
+ ]);
+
+ expect(results).toEqual([null, null, null]);
+ expect(pool.timedOutFiles()).toEqual(["t.ts"]);
+ expect(pool.crashedTasks()).toBe(0);
+ await pool.destroy();
+ });
});
diff --git a/ix-cli/src/cli/__tests__/stale-workspace.test.ts b/ix-cli/src/cli/__tests__/stale-workspace.test.ts
index 0d931555..d7d97342 100644
--- a/ix-cli/src/cli/__tests__/stale-workspace.test.ts
+++ b/ix-cli/src/cli/__tests__/stale-workspace.test.ts
@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
+import { execFileSync } from "node:child_process";
import { clearIngestMtimeCache, clearMapBaseline, ingestMtimeCachePath, saveConfig } from "../config.js";
import { loadIngestBaseline } from "../ingest-baseline.js";
@@ -77,6 +78,7 @@ describe("workspace-scoped staleness", () => {
staleFiles: 1,
sampleChangedFiles: ["a.js"],
replayedFiles: [],
+ parseTimeouts: [],
});
expect(
@@ -141,6 +143,7 @@ describe("workspace-scoped staleness", () => {
staleFiles: 0,
sampleChangedFiles: [],
replayedFiles: [],
+ parseTimeouts: [],
});
// The workspace is unverified, but no individual file is known to have
// changed — the distinction this pair of assertions exists to pin down.
@@ -203,6 +206,7 @@ describe("workspace-scoped staleness", () => {
staleFiles: 0,
sampleChangedFiles: [],
replayedFiles: [],
+ parseTimeouts: [],
});
});
@@ -276,6 +280,7 @@ describe("workspace-scoped staleness", () => {
staleFiles: 1,
sampleChangedFiles: ["deleted.js"],
replayedFiles: [],
+ parseTimeouts: [],
});
// From inside the workspace, as a read runs. The probe resolves the active
// root from cwd, and a repository cwd now outranks `default: true` (see
@@ -317,6 +322,8 @@ describe("workspace-scoped staleness", () => {
tracksMapBaseline: true,
extractor: "tree-sitter/9.9",
replayedFiles: [],
+ pendingFiles: [],
+ parseTimeouts: [],
});
});
@@ -340,6 +347,67 @@ describe("workspace-scoped staleness", () => {
expect(loadIngestBaseline(root)?.currentRev).toBe(20);
});
+ it("judges a root reached through a symlink by its own relative paths", () => {
+ // macOS's tmpdir (/var -> /private/var) and Windows 8.3 names are roots
+ // whose canonical path differs. Discovery returns canonical paths; the
+ // baseline may hold either spelling, and the report must say `a.js`.
+ const realRoot = path.join(home, "real");
+ const linkedRoot = path.join(home, "linked");
+ writeSource(realRoot, "a.js", "export const a = 1;\n");
+ writeSource(realRoot, "b.js", "export const b = 1;\n");
+ writeSource(realRoot, "gone.js", "export const gone = 1;\n");
+ fs.symlinkSync(realRoot, linkedRoot, "junction");
+ const fileA = path.join(linkedRoot, "a.js");
+ const fileB = fs.realpathSync.native(path.join(realRoot, "b.js"));
+ const gone = path.join(linkedRoot, "gone.js");
+
+ expect(
+ persistIngestBaselineIfClean(
+ linkedRoot,
+ new Map([
+ [fileA, fs.statSync(fileA).mtimeMs],
+ [fileB, fs.statSync(fileB).mtimeMs],
+ [gone, fs.statSync(gone).mtimeMs],
+ ]),
+ 3,
+ 0,
+ 0,
+ new Date(Date.now() + 60_000),
+ ),
+ ).toBe(true);
+ expect(detectStaleFiles(linkedRoot).staleFiles).toBe(0);
+
+ moveMtimeForward(fileA);
+ fs.rmSync(gone);
+ const result = detectStaleFiles(linkedRoot);
+ expect(result.staleFiles).toBe(2);
+ expect([...result.sampleChangedFiles].sort()).toEqual(["a.js", "gone.js"]);
+ });
+
+ it("stops a walk at its cap without calling the files past it deleted; a git listing is not capped", () => {
+ // Status runs on every agent turn. Without a cap, a walk of a directory
+ // that is not a work tree -- a home directory, say -- reads the whole disk.
+ const root = path.join(home, "capped");
+ const files = ["a.js", "b.js", "c.js", "d.js", "e.js"].map((name) => writeSource(root, name, ""));
+ persistIngestBaselineIfClean(
+ root,
+ new Map(files.map((f) => [f, fs.statSync(f).mtimeMs])),
+ 4,
+ 0,
+ 0,
+ new Date(Date.now() + 60_000),
+ );
+ for (const f of files) moveMtimeForward(f);
+
+ expect(detectStaleFiles(root, 5).staleFiles, "uncapped, every edit is seen").toBe(5);
+ const capped = detectStaleFiles(root, 5, 3);
+ expect(capped.staleFiles, "only the files read, and none reported deleted").toBe(3);
+
+ execFileSync("git", ["init", "-q"], { cwd: root, stdio: "ignore" });
+ execFileSync("git", ["add", "-A"], { cwd: root, stdio: "ignore" });
+ expect(detectStaleFiles(root, 5, 3).staleFiles, "git lists the whole repository").toBe(5);
+ });
+
it("uses ingest time for files absent from the mtime cache", async () => {
const root = path.join(home, "partial-cache");
const mappedFile = writeSource(root, "mapped.js", "export const mapped = true;\n");
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index d33b5d28..937d0af0 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -4,7 +4,6 @@ import * as nodePath from 'node:path';
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import * as os from 'node:os';
-import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import type { Command } from 'commander';
import { ParsePool } from './parse-pool.js';
@@ -14,9 +13,14 @@ import { IxClient } from '../../client/api.js';
import type { GraphPatchPayload } from '../../client/types.js';
import { canonicalWorkspacePath, isPathInside, resolveWorkspaceRoot, clearMapResultCache, clearStitchScopeCache } from '../config.js';
import { resolveIngestRoot } from '../map-root.js';
+import { acquireMapLock, requestMapRerun, takeMapRerun } from '../single-flight.js';
+import { applyRequestedMapCoalesceExitCode } from './map.js';
+import {
+ canonicalizeDiscoveredFilePath, discoverSourceFiles, emptyDiscoveryCounts, isSupportedSourceFile,
+} from '../file-discovery.js';
import {
clearRebuildProgress, extractorChanged, isRev, loadIngestBaseline, loadRebuildProgress,
- saveIngestBaseline, saveRebuildProgress,
+ saveIngestBaseline, saveRebuildProgress, type BaselineFileNotes,
} from '../ingest-baseline.js';
import { resolveGitHubToken } from '../github/auth.js';
import { parseGitHubRepo, fetchGitHubData } from '../github/fetch.js';
@@ -35,10 +39,8 @@ import {
perFileAction,
} from '../commit-breaker.js';
import { readBackendHealth } from './upgrade.js';
-import { SUPPORTED_EXTENSIONS } from '../supported-extensions.js';
import {
- createIgnoreMatcher, parseIgnoreFile, parseIgnorePattern,
- type IgnoreMatcher, type IgnorePattern,
+ createIgnoreMatcher, parseIgnoreFile, parseIgnorePattern, type IgnorePattern,
} from '../ignore-globs.js';
import { canRenderProgress } from '../stderr.js';
import { createTypeScriptModuleResolver } from '../ts-module-resolution.js';
@@ -57,14 +59,7 @@ import { createClient } from "../../client/factory.js";
// File discovery
// ---------------------------------------------------------------------------
-export function isSupportedSourceFile(filePath: string): boolean {
- const fileName = nodePath.basename(filePath).toLowerCase();
- return fileName === 'dockerfile'
- || fileName.endsWith('.dockerfile')
- || fileName === 'makefile'
- || fileName === 'gnumakefile'
- || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
-}
+export { isSupportedSourceFile, tryGitLsFiles, walkFiles } from '../file-discovery.js';
// Extensions whose source text must be pre-read so buildGlobalResolutionIndex can
// extract cross-batch symbols before the streaming parse loop. Go uses a fast
@@ -107,64 +102,29 @@ function parseLangs(raw: string): Set<string> {
);
}
-const IGNORE_DIRS = new Set([
- 'node_modules', '.git', '.hg', '.svn', 'dist', 'build', 'target', 'out', '.next',
- '__pycache__', '.tox', '.venv', 'venv', '.mypy_cache', '.pytest_cache',
- '.gradle', '.idea', '.vscode', '.settings', '.vs',
- '.cache', '.parcel-cache', 'coverage', '.nyc_output',
- 'vendor', 'Pods', '.dart_tool', '.pub-cache',
- 'bin', 'obj', 'pkg',
- '.ix', '.claude', '.gitnexus',
- 'test', 'tests', '__tests__', 'spec', 'specs', 'e2e',
- 'examples', 'fixtures', '__mocks__', '__fixtures__',
-]);
-
-// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.
-// These files inflate parse time and graph size without contributing to architectural signal.
-const GENERATED_FILE_SUFFIXES = [
- '.pb.go', // protobuf generated Go
- '_deepcopy.go', // controller-gen deepcopy (Kubernetes ecosystem)
- '_mock.go', // gomock generated mocks
- '.pb.ts', // protobuf generated TypeScript
- '.pb.js', // protobuf generated JavaScript
- '_pb.ts', // alternative protobuf TypeScript naming
- '_pb.js', // alternative protobuf JavaScript naming
-];
-const GENERATED_FILE_PREFIXES = [
- 'zz_generated', // controller-gen convention (Kubernetes ecosystem)
- 'mock_', // gomock convention
-];
+const MAX_FILE_BYTES = 1024 * 1024; // 1 MB
-// Generated dependency lockfiles, matched whole. Only the ones whose extension
-// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,
-// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end
-// in `.lock`, which is not a discovered extension, so they never reach here.
-// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger
-// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and
-// it describes a resolved dependency snapshot, not architecture.
-const GENERATED_FILE_NAMES = new Set([
- 'package-lock.json', // npm
- 'npm-shrinkwrap.json', // npm, published form
- 'pnpm-lock.yaml', // pnpm
- 'packages.lock.json', // NuGet
-]);
+/** Parse workers at most, unless IX_PARSE_WORKERS says otherwise. */
+const DEFAULT_PARSE_WORKERS = 8;
+/** Files a parse worker should have before another one is worth starting. */
+const FILES_PER_PARSE_WORKER = 50;
-function isGeneratedFile(basename: string): boolean {
- // Lowercased for the name match alone: `isSupportedSourceFile` lowercases
- // too, so on a case-insensitive filesystem a `Package-lock.json` is
- // discovered and would otherwise slip past a case-sensitive comparison.
- if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;
- for (const suffix of GENERATED_FILE_SUFFIXES) {
- if (basename.endsWith(suffix)) return true;
- }
- for (const prefix of GENERATED_FILE_PREFIXES) {
- if (basename.startsWith(prefix)) return true;
- }
- return false;
+/**
+ * How many parse workers `expectedTasks` files deserve: one per 50 files,
+ * never more than the cores less one or IX_PARSE_WORKERS (default 8), never
+ * fewer than one. Each worker loads every grammar, about 70 MB, so a worker
+ * per core for a one-file edit cost more than the parse.
+ */
+export function parsePoolSize(
+ expectedTasks: number,
+ cores: number = os.availableParallelism(),
+ configured: string | undefined = process.env.IX_PARSE_WORKERS,
+): number {
+ const fromEnv = Number(configured);
+ const cap = configured !== undefined && Number.isInteger(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_PARSE_WORKERS;
+ return Math.max(1, Math.min(cores - 1, cap, Math.ceil(expectedTasks / FILES_PER_PARSE_WORKER)));
}
-
-const MAX_FILE_BYTES = 1024 * 1024; // 1 MB
const MINIFIED_BYTES_THRESHOLD = 200 * 1024;
const MINIFIED_MAX_LINE_THRESHOLD = 20_000;
const MINIFIED_AVG_LINE_THRESHOLD = 2_000;
@@ -181,32 +141,6 @@ export function parsePositiveIntEnv(name: string, fallback: number): number {
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
-export function* walkFiles(
- dir: string,
- recursive: boolean,
- exclude?: { matcher: IgnoreMatcher; root: string; onSkip?: () => void },
-): Generator<string> {
- let entries: fs.Dirent[];
- try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
- catch { return; }
-
- for (const entry of entries) {
- if (entry.isDirectory() && entry.name.startsWith('.') && entry.name !== '.') continue;
- if (IGNORE_DIRS.has(entry.name)) continue;
- const full = nodePath.join(dir, entry.name);
- // Directories are tested before descending, so an excluded tree costs one
- // check rather than one per file inside it.
- if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), entry.isDirectory())) {
- exclude.onSkip?.();
- continue;
- }
- if (entry.isDirectory()) {
- if (recursive) yield* walkFiles(full, true, exclude);
- } else if (entry.isFile()) {
- if (isSupportedSourceFile(entry.name) && !isGeneratedFile(entry.name)) yield full;
- }
- }
-}
/**
* The patterns in force for this ingest: `--exclude` flags first, then the
@@ -229,13 +163,6 @@ export function collectExcludePatterns(root: string, cliPatterns: string[] = [])
return patterns;
}
-function canonicalizeDiscoveredFilePath(filePath: string): string {
- try {
- return fs.realpathSync.native(filePath);
- } catch {
- return nodePath.resolve(filePath);
- }
-}
/**
* True when `candidate` lies inside `root`.
@@ -307,67 +234,11 @@ export function discoverIngestFilePaths(
// root drops everything whenever the root is itself reached through a link —
// macOS `/var` -> `/private/var`, a home on a network mount, a `~/code`
// symlink — which would look like "this repo has no source files".
- const canonicalRoot = canonicalize(root);
+ const canonicalRoot = canonicalizeDiscoveredFilePath(root);
const files = canonical.filter((candidate) => isWithinDiscoveryRoot(canonicalRoot, candidate));
return { files, outsideRoot: canonical.length - files.length };
}
-/**
- * Exported for the discovery tests: the symlink-containment guarantee is a
- * property of git-listing -> canonicalize -> confine *composed*, and each step
- * looks fine alone. A test that cannot run the real listing cannot show that a
- * committed symlink escapes in the first place.
- */
-export function tryGitLsFiles(
- dir: string,
- recursive: boolean,
- exclude?: { matcher: IgnoreMatcher; root: string; onSkip?: () => void },
-): string[] | null {
- try {
- const result = spawnSync(
- 'git',
- // -c: a cloned repository's own .git/config can name a core.fsmonitor
- // command, which ls-files would run. Never let it.
- ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', 'ls-files', '--cached', '--others', '--exclude-standard'],
- {
- cwd: dir,
- encoding: 'utf-8',
- maxBuffer: 32 * 1024 * 1024,
- }
- );
- if (result.status !== 0) return null;
-
- const files: string[] = [];
- const seenCanonical = new Set<string>();
- for (const line of result.stdout.split(/\r?\n/)) {
- if (!line) continue;
- const fullPath = nodePath.resolve(dir, line);
- if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
- // Before the extension check, so the count reports what the caller
- // excluded rather than what they excluded and Ix would have skipped
- // anyway. `git ls-files` never saw IGNORE_DIRS either, which is why a
- // git repository's `tests/` tree reaches the graph today.
- if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {
- exclude.onSkip?.();
- continue;
- }
- if (!isSupportedSourceFile(fullPath)) continue;
- if (isGeneratedFile(nodePath.basename(fullPath))) continue;
- try {
- if (!fs.statSync(fullPath).isFile()) continue;
- const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);
- if (seenCanonical.has(canonicalPath)) continue;
- seenCanonical.add(canonicalPath);
- files.push(canonicalPath);
- } catch {
- // Ignore racy deletes and permission issues during discovery.
- }
- }
- return files;
- } catch {
- return null;
- }
-}
function sha256(content: Buffer): string {
return crypto.createHash('sha256').update(content).digest('hex');
@@ -971,9 +842,25 @@ export function persistIngestBaselineIfClean(
now?: Date,
deletedFiles: Map<string, string[]> = new Map(),
extractor?: string | null,
- replayedFiles: readonly string[] = [],
+ notes: BaselineFileNotes = {},
): boolean {
if (!ingestCompletedCleanly(parseErrors, commitErrors)) return false;
+ return persistIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);
+}
+
+/**
+ * Write the baseline -- whatever errors the run had. The caller has already
+ * left the files it did not settle at their previous mtimes.
+ */
+export function persistIngestBaseline(
+ projectRoot: string,
+ mtimes: Map<string, number>,
+ currentRev: number,
+ now?: Date,
+ deletedFiles: Map<string, string[]> = new Map(),
+ extractor?: string | null,
+ notes: BaselineFileNotes = {},
+): boolean {
// An empty mtime map is normally a discovery failure, not an empty repo —
// an over-broad ignore rule, the wrong cwd, a glob that matched nothing —
// and persisting it silently discards the baseline, forcing a full re-ingest
@@ -985,7 +872,7 @@ export function persistIngestBaselineIfClean(
// empty is a real state and is persisted; without them, it is still treated
// as discovery having gone wrong.
if (mtimes.size === 0 && deletedFiles.size === 0) return false;
- saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, replayedFiles);
+ saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles, extractor, notes);
return true;
}
@@ -1013,6 +900,76 @@ function renderProgressLine(phase: string, current: number, total: number): stri
// Command registration
// ---------------------------------------------------------------------------
+type PathIngestOpts = Parameters<typeof ingestFiles>[1];
+
+/** The workspace root `target` ingests into, or undefined when it cannot be resolved. */
+function ingestLockRoot(target: string, explicitRoot?: string): string | undefined {
+ try {
+ const resolved = nodePath.resolve(target);
+ return resolveIngestRoot(resolved, fs.statSync(resolved).isDirectory(), explicitRoot);
+ } catch {
+ // `ingestFiles` reports a missing or unreadable path itself.
+ return undefined;
+ }
+}
+
+/**
+ * `ix ingest <path>`, single-flight per workspace under the lock `ix map`
+ * holds. Two ingests (or an ingest and a map) over one workspace race on the
+ * mtime baseline: the second to finish overwrites what the first recorded.
+ *
+ * A run that finds the lock held coalesces the way `ix map` does: it asks the
+ * holder for one more pass (`requestMapRerun`) and exits 0, unless
+ * IX_MAP_COALESCE_EXIT_CODE asks for another code. It used to exit 1, which
+ * editor hooks and MCP clients that ingest after every edit reported as a
+ * failed ingest. The holder takes the request before it lets go: `ix map`
+ * re-ingests the workspace, and so does a holding `ix ingest` when the
+ * workspace has a baseline -- an incremental pass, so it costs a stat per
+ * file. Without a baseline that pass would be a first ingest of the whole
+ * workspace, so it re-ingests only its own path then.
+ */
+export async function ingestPathSingleFlight(target: string, opts: PathIngestOpts): Promise<void> {
+ const root = ingestLockRoot(target, opts.root);
+ if (root === undefined) {
+ await ingestFiles(target, opts);
+ return;
+ }
+ const lock = acquireMapLock(root, `ix ingest ${root}`);
+ if (lock === null) {
+ requestMapRerun(root);
+ if (opts.format === 'json') {
+ // A JSON consumer gets an object, not an empty stdout it cannot parse.
+ printJson({ coalesced: true, workspace: root });
+ } else {
+ process.stderr.write(chalk.dim(' Another ix map or ix ingest is running for this workspace; it will pick up this change before it exits.\n'));
+ }
+ applyRequestedMapCoalesceExitCode();
+ return;
+ }
+ // Holding the lock: a request left before now is covered by this run, which
+ // has not read the tree yet.
+ takeMapRerun(root);
+ try {
+ await ingestFiles(target, opts);
+ if (takeMapRerun(root)) {
+ // One pass at most, as in `ix map`. The waiting run's own flags are not
+ // known here, so this is a plain incremental pass.
+ const rerunTarget = loadIngestBaseline(root) !== null ? root : target;
+ try {
+ await ingestFiles(rerunTarget, {
+ recursive: true, format: 'json', printSummary: false, suppressOutput: true, root: opts.root,
+ });
+ } catch (err: any) {
+ if (opts.format !== 'json') {
+ process.stderr.write(chalk.dim(` Re-run for an ingest that waited on this one failed: ${err?.message ?? err}\n`));
+ }
+ }
+ }
+ } finally {
+ lock.release();
+ }
+}
+
export function registerIngestCommand(program: Command): void {
program
.command('ingest [path]')
@@ -1044,7 +1001,7 @@ export function registerIngestCommand(program: Command): void {
if (opts.github) {
await ingestGitHub(opts);
} else if (effectivePath) {
- await ingestFiles(effectivePath, opts);
+ await ingestPathSingleFlight(effectivePath, opts);
} else {
console.error('Error: provide a <path> or use --github <owner/repo>');
process.exit(1);
@@ -1072,6 +1029,14 @@ const COMMIT_STATUS_IDEMPOTENT = 'Idempotent';
*/
const COMMIT_STATUS_BASE_REV_MISMATCH = 'BaseRevMismatch';
+/** The stderr note for files skipped because their parse ran past the budget. */
+export function describeParseTimeouts(files: readonly string[], sample = 5): string {
+ const shown = files.slice(0, sample).join(", ");
+ const more = files.length > sample ? ` and ${files.length - sample} more` : "";
+ return `Skipped ${files.length} file(s) whose parse ran past the per-file budget: ${shown}${more}. ` +
+ "They are not in the graph. Raise the budget with IX_PARSE_BUDGET_MS (milliseconds, 0 = none).";
+}
+
/** Minimal local-ingest facts needed by commands that continue after ingestion. */
export interface IngestFilesSummary {
filesDiscovered: number;
@@ -1158,6 +1123,12 @@ export interface IngestFilesSummary {
* something must not claim the graph is where it was.
*/
graphUnchanged: boolean;
+ /**
+ * Files skipped because their parse ran past the per-file budget
+ * (IX_PARSE_BUDGET_MS, default 10 s), workspace-relative. A subset of the
+ * unparsed skips, named because a file that times out does so on every run.
+ */
+ parseTimeouts: string[];
}
/**
@@ -1297,8 +1268,8 @@ export function describeStitchSkipped(
// stay frozen. Only a run that re-ingests every file gets back in.
// Every branch names `--force`, because nothing else re-registers this
// workspace. A skipped stitch does NOT stop the run persisting its mtime
- // baseline -- `persistIngestBaselineIfClean` gates on parse and commit errors
- // only -- so an ordinary re-run is incremental, skips unchanged files, and
+ // baseline -- the baseline records every file the run settled, stitch or
+ // not -- so an ordinary re-run is incremental, skips unchanged files, and
// never enters the stitch block. "Re-run once that finishes" and "raise
// IX_MAP_DEADLINE_MS" were both promises this code cannot keep: the re-run
// prints nothing, exits 0, and the cross-repo edges stay exactly as stale.
@@ -1784,10 +1755,22 @@ export async function ingestFiles(
* narrowing makes `pool` `never` down there.
*/
const crashedParses = (): number => (pool === null ? 0 : pool.crashedTasks());
- const ensureParsePool = (): ParsePool => {
- if (pool) return pool;
- pool = new ParsePool(workerPath, Math.max(1, os.cpus().length - 1));
- pool.init();
+ /** Files whose parse ran past the budget; see `ParsePool.timedOutFiles`. */
+ const timedOutParses = (): string[] => (pool === null ? [] : pool.timedOutFiles());
+ /**
+ * The pool, sized to the work the caller is about to give it: one worker per
+ * 50 files, capped by the cores and by IX_PARSE_WORKERS (default 8). A
+ * one-file edit used to start a worker per core -- 23 threads and about
+ * 1.6 GB on a 24-core machine -- to parse one file. Workers start lazily, so
+ * the size is a ceiling, and a later, larger phase can raise it.
+ */
+ const ensureParsePool = (expectedTasks: number): ParsePool => {
+ const size = parsePoolSize(expectedTasks);
+ if (pool) {
+ pool.growTo(size);
+ return pool;
+ }
+ pool = new ParsePool(workerPath, size);
return pool;
};
@@ -1824,7 +1807,7 @@ export async function ingestFiles(
// streaming loop below. A loss here is not a file the RUN lost, and
// counting it as one refused the stitch and withheld the baseline over
// files that were all present.
- chunk.map(fp => ensureParsePool().parse(fp, relSources.get(fp)!, false).catch(() => null)),
+ chunk.map(fp => ensureParsePool(targets.length).parse(fp, relSources.get(fp)!, false).catch(() => null)),
);
for (let j = 0; j < chunk.length; j++) {
if (parsed[j]) preParsed.set(chunk[j], parsed[j]);
@@ -1867,11 +1850,21 @@ export async function ingestFiles(
* is told about is indistinguishable from an ingest that missed something.
*/
let filesExcluded = 0;
+ /** What discovery left out: SKIPPED_DIRS hits and files it could not stat. */
+ const discoveryCounts = emptyDiscoveryCounts();
let filesChanged = 0;
let patchesApplied = 0;
let idempotentPatches = 0;
/** Workspace-relative paths of changed files the backend answered `Idempotent` (F-01). */
const replayedChanges = new Set<string>();
+ // Per-file outcomes, so one failing file does not keep the whole baseline
+ // from being written. A changed file is settled once its patch is committed
+ // (`committedRel`), its content turns out to be what the backend has, or it
+ // is skipped for a reason that will not change (minified); a file that
+ // parsed to nothing is settled only if no worker crashed this run.
+ const committedRel = new Set<string>();
+ const settledAbs = new Set<string>();
+ const unparsedRel = new Set<string>();
// What `graphUnchanged` is computed from, beside the counters above. Each is
// a way this run can have written to the graph, or have found it not where
// the last run left it, without a patch being counted.
@@ -2051,12 +2044,14 @@ export async function ingestFiles(
const exclude = excludeMatcher.size > 0
? { matcher: excludeMatcher, root: workspaceRoot, onSkip: () => { filesExcluded += 1; } }
: undefined;
+ // Discovery returns canonical paths, so confinement below does not resolve
+ // every file a second time; only the root is canonicalized there.
const discovery = discoverIngestFilePaths(
stat.isFile()
- ? (isSupportedSourceFile(scopePath) ? [scopePath] : [])
- : (tryGitLsFiles(scopePath, opts.recursive ?? true, exclude)
- ?? Array.from(walkFiles(scopePath, opts.recursive ?? true, exclude))),
+ ? (isSupportedSourceFile(scopePath) ? [canonicalizeDiscoveredFilePath(scopePath)] : [])
+ : discoverSourceFiles(scopePath, { recursive: opts.recursive ?? true, exclude, counts: discoveryCounts }),
stat.isFile() ? undefined : scopePath,
+ (p) => p,
);
// `--lang` narrows discovery to the named languages. The files it leaves
// out keep their baseline entries (`langExcluded`, below): this run says
@@ -2075,9 +2070,9 @@ export async function ingestFiles(
// scope is read or parsed.
const resolutionPaths = scopedRun
? discoverIngestFilePaths(
- tryGitLsFiles(workspaceRoot, true, exclude && { ...exclude, onSkip: undefined })
- ?? Array.from(walkFiles(workspaceRoot, true, exclude && { ...exclude, onSkip: undefined })),
+ discoverSourceFiles(workspaceRoot, { exclude: exclude && { ...exclude, onSkip: undefined } }),
workspaceRoot,
+ (p) => p,
).files
: filePaths;
const resolveOpts = {
@@ -2176,10 +2171,23 @@ export async function ingestFiles(
// DB-reset guard: if the mtime cache has entries but the server returns no hashes
// for a small sample, the DB was wiped (e.g. `ix reset` run from a different dir
// so the cache wasn't cleared). Invalidate the cache so files are re-ingested.
+ // Every baseline entry was committed (a file that failed keeps its previous
+ // entry or none), so each sampled file should be known to the backend.
+ // The sample is spread across the baseline rather than its first five
+ // entries, and a lookup that FAILED proves nothing: it used to come back
+ // empty and read as a reset, re-ingesting the whole workspace over one
+ // timed-out request.
if (!opts.force && mtimeCache.size > 0) {
- const samplePaths = [...mtimeCache.keys()].slice(0, 5);
- const sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug);
- if (sampleHashes.size === 0) {
+ const keys = [...mtimeCache.keys()];
+ const step = Math.max(1, Math.floor(keys.length / 5));
+ const samplePaths = keys.filter((_, i) => i % step === 0).slice(0, 5);
+ let sampleHashes: Map<string, string> | undefined;
+ try {
+ sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug, true);
+ } catch (err) {
+ if (debug) process.stderr.write(`\n DB-reset check skipped: hash lookup failed: ${err}\n`);
+ }
+ if (sampleHashes !== undefined && sampleHashes.size === 0) {
mtimeCache.clear();
// What an unfinished re-ingest committed is gone with the rest.
rebuildProgress?.clear();
@@ -2364,6 +2372,7 @@ export async function ingestFiles(
replayedChanges.add(item.filePath);
return;
}
+ committedRel.add(item.filePath);
onCommitted?.(item, rev);
};
@@ -2911,9 +2920,10 @@ export async function ingestFiles(
// `BaseRevMismatch` wrote nothing: the backend read the latest rev
// outside the transaction and it moved before the commit ran. It
// is a lost race, not a replay, so it must not count as applied —
- // and it belongs in commitErrors, because that is what stops
- // `persistIngestBaselineIfClean` caching these files as unchanged
- // and leaving them missing from the graph until the next edit.
+ // it belongs in commitErrors, and it must not reach
+ // `settleCommitted`: an unsettled file keeps its previous mtime,
+ // which is what stops the baseline caching it as unchanged and
+ // leaving it missing from the graph until the next edit.
if (result.status === COMMIT_STATUS_BASE_REV_MISMATCH) {
commitErrors += items.length;
if (debug) {
@@ -3224,9 +3234,26 @@ export async function ingestFiles(
}
};
- if ((knownHashes.size > 0 || mtimeChangedPaths.length === 0) && !opts.force) {
+ // A local baseline with mtimes is enough to take the incremental path, even
+ // when the backend knows none of the changed files: adding one new file to a
+ // mapped repo is exactly that, and it used to re-send the whole repository.
+ // `mtimeCache` is empty after --force, a workspace migration or the DB-reset
+ // guard above, so those still take Path B.
+ //
+ // Only for files whose cross-file references Path A can resolve, though.
+ // Path A resolves the changed files as one batch against `globalIndex`, and
+ // that index holds the symbols of the prescan languages only (Go, PHP, R,
+ // SAS, JS/TS). A new Python or Java file resolved that way binds its calls
+ // to itself alone, so they point at a module node or at nothing; the
+ // whole-repository pass resolves them. Those files keep that pass until the
+ // persisted symbol table (IN-10) covers every language.
+ const unknownToBackend = mtimeChangedPaths.filter(fp => !knownHashes.has(fp));
+ const hasBaseline = previousBaseline !== null && mtimeCache.size > 0
+ && unknownToBackend.every(needsIndexPrescan);
+ if ((hasBaseline || knownHashes.size > 0 || mtimeChangedPaths.length === 0) && !opts.force) {
// Path A: has baseline or all mtime-clean → pre-scan to detect changes before loading modules.
- // If nothing changed, module load is skipped entirely.
+ // If nothing changed, module load is skipped entirely. A file with no
+ // backend hash is new: it is parsed and sent with no reconcile.
const changedPaths: Array<{ filePath: string; bytes: Buffer; hash: string; previousHash: string | undefined }> = [];
for (const filePath of mtimeChangedPaths) {
try {
@@ -3235,6 +3262,7 @@ export async function ingestFiles(
if (!forceReingestPaths.has(filePath) && knownHashes.get(filePath) === hash) {
filesSkipped++;
filesSkippedAsUnchanged++;
+ settledAbs.add(filePath);
progressCurrent++;
continue;
}
@@ -3263,6 +3291,7 @@ export async function ingestFiles(
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
minifiedLikely++;
+ settledAbs.add(filePath);
filesSkipped++;
progressCurrent++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
@@ -3319,13 +3348,13 @@ export async function ingestFiles(
// paths. f.absFilePath is retained only for debug/error display.
const parseResults = await Promise.all(
chunk.map(f =>
- ensureParsePool().parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
+ ensureParsePool(parseable.length).parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
),
);
const batch: ParsedFile[] = [];
for (let j = 0; j < chunk.length; j++) {
const parsed = parseResults[j] as any;
- if (!parsed) { filesSkipped++; filesSkippedUnparsed++; continue; }
+ if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(chunk[j].filePath); continue; }
entitiesParsed += parsed.entities.length;
batch.push({ filePath: chunk[j].filePath, parsed, hash: chunk[j].hash, previousHash: chunk[j].previousHash });
}
@@ -3335,18 +3364,17 @@ export async function ingestFiles(
await pendingFlush;
}
} else {
- // Path B: no baseline (first ingest) or --force → load modules, then stream parse + commit.
+ // Path B: no baseline (first ingest, migrated workspace, wiped backend) or
+ // --force → load modules, then stream parse + commit.
//
// Ix#568: this branch walks ALL of `filePaths`, re-reading, re-hashing and
// re-parsing every one, and its hash-clean short-circuit cannot fire here
// -- `knownHashes` is empty by construction on the non-force entry, and
// the check is `!opts.force` on the other. So the mtime skips the stat
- // loop recorded did not actually happen, and leaving them counted refused
- // the stitch on the commonest incremental shape there is: add ONE new
- // file to a mapped repo, and `loadExistingHashes` returns nothing for it,
- // so `knownHashes.size === 0` sends the whole repo down here -- every file
- // parsed, registration complete -- while the gate still saw N-1 files
- // "skipped as unchanged" and silently declined to stitch.
+ // loop recorded did not actually happen, and leaving them counted would
+ // refuse the stitch on a run that parsed every file. (Adding one file to
+ // a mapped repo used to land here whatever its language; the baseline
+ // check above keeps prescan-language additions on Path A now.)
filesSkipped -= filesSkippedAsUnchanged;
filesSkippedAsUnchanged = 0;
const moduleStart = performance.now();
@@ -3466,12 +3494,14 @@ export async function ingestFiles(
if (!opts.force && !forceReingestPaths.has(absFilePath) && knownHashes.get(absFilePath) === hash) {
filesSkipped++;
filesSkippedAsUnchanged++;
+ settledAbs.add(absFilePath);
return;
}
const previousHash = knownHashes.get(absFilePath);
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
minifiedLikely++;
+ settledAbs.add(absFilePath);
filesSkipped++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${absFilePath}\n`);
return;
@@ -3479,7 +3509,7 @@ export async function ingestFiles(
const relFilePath = toWorkspaceRelative(absFilePath);
const fd: NonNullable<FileData> = { filePath: relFilePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };
fileData[idx] = fd;
- parsePromises[idx] = ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });
+ parsePromises[idx] = ensureParsePool(filePaths.length).parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });
} catch (err) {
parseErrors++;
process.stderr.write(`\n [read error] ${absFilePath}: ${err}\n`);
@@ -3493,7 +3523,7 @@ export async function ingestFiles(
const f = fileData[j];
if (!f) continue;
const parsed = parseResults[j] as any;
- if (!parsed) { filesSkipped++; filesSkippedUnparsed++; continue; }
+ if (!parsed) { filesSkipped++; filesSkippedUnparsed++; unparsedRel.add(f.filePath); continue; }
entitiesParsed += parsed.entities.length;
batch.push({ filePath: f.filePath, parsed, hash: f.hash, previousHash: f.previousHash });
}
@@ -3570,17 +3600,26 @@ export async function ingestFiles(
const recovery = pendingDeletionRecovery.get(item.filePath);
if (!recovery || !previousBaseline) return;
durableDeletedFiles.set(recovery.absolutePath, recovery.dependents);
- saveIngestBaseline(
- projectRoot,
- previousMtimes,
- previousBaseline.currentRev,
- new Date(previousBaseline.lastIngestAt),
- durableDeletedFiles,
- previousBaseline.extractor,
- previousBaseline.replayedFiles,
- );
},
});
+ // Once, after the deletions -- it was once per deleted file, the whole
+ // baseline rewritten each time. The run's own baseline write follows;
+ // this records the restore hints in case the run dies before then.
+ if (previousBaseline && durableDeletedFiles.size > 0) {
+ saveIngestBaseline(
+ projectRoot,
+ previousMtimes,
+ previousBaseline.currentRev,
+ new Date(previousBaseline.lastIngestAt),
+ durableDeletedFiles,
+ previousBaseline.extractor,
+ {
+ replayedFiles: previousBaseline.replayedFiles,
+ pendingFiles: previousBaseline.pendingFiles,
+ parseTimeouts: previousBaseline.parseTimeouts,
+ },
+ );
+ }
}
}
@@ -3618,33 +3657,67 @@ export async function ingestFiles(
return abs !== undefined && (!inScope(abs) || langExcluded.includes(abs));
}),
];
- const baselinePersisted = (scopedRun && !previousBaseline) ? false : persistIngestBaselineIfClean(
+ // Per file, not all or nothing. The baseline used to be written only when
+ // the whole run was clean, so one permanently unreadable file meant no run
+ // ever recorded anything again: every map re-read the whole workspace and
+ // deletions were never retracted. Now every file this run settled is
+ // recorded, and every changed file it did not settle -- a read or build
+ // error, a commit that failed, a parse lost to a dead worker -- keeps its
+ // previous mtime (or none) so the next run tries it again. Lost parses
+ // still count: a run whose worker pool died resolves every later file as
+ // null, and recording those would skip them as unchanged for good.
+ const runClean = ingestCompletedCleanly(parseErrors + crashedParses(), commitErrors);
+ const lostParses = crashedParses() > 0;
+ // A parse that ran past the budget is not settled either, though it is
+ // not a crash: the file is not in the graph. It used to be recorded clean,
+ // and since a slow file times out on every run, it was then skipped as
+ // unchanged for good while status called the graph current.
+ const timedOut = new Set(timedOutParses());
+ const unsettled = mtimeChangedPaths.filter(abs => {
+ if (settledAbs.has(abs)) return false;
+ const rel = toWorkspaceRelative(abs);
+ if (timedOut.has(rel)) return true;
+ return !committedRel.has(rel) && !(unparsedRel.has(rel) && !lostParses);
+ });
+ // Name the unsettled files too. A new one has no previous mtime, so it is
+ // left out of the baseline, and `ix status` judges a file the baseline
+ // does not list against `lastIngestAt` -- which this write moves past the
+ // file's mtime. Without the list, status called the graph current while
+ // the file was missing from it. A replayed change has its own list. As
+ // with replays, a run that did not look at a file has not resolved it.
+ const notLookedAt = (rel: string): boolean => {
+ const abs = nodePath.resolve(workspaceRoot, rel);
+ return (!inScope(abs) || langExcluded.includes(abs)) && fs.existsSync(abs);
+ };
+ const pendingFiles = [
+ ...unsettled.map(toWorkspaceRelative).filter(rel => !replayedChanges.has(rel) && !timedOut.has(rel)),
+ ...(previousBaseline?.pendingFiles ?? []).filter(notLookedAt),
+ ];
+ // Timeouts get their own list, which `ix status` warns about by name: the
+ // fix is a larger IX_PARSE_BUDGET_MS, not another run.
+ const parseTimeouts = [...timedOut, ...(previousBaseline?.parseTimeouts ?? []).filter(notLookedAt)];
+ // A deletion whose patch did not land keeps its entry, so it is found
+ // deleted again next run.
+ const failedDeletions = deletedPaths.filter(abs => !committedRel.has(toWorkspaceRelative(abs)));
+ const baselinePersisted = (scopedRun && !previousBaseline) ? false : persistIngestBaseline(
projectRoot,
retryOnNextRun(
- carryForwardMtimes(currentMtimes, previousMtimes, [...deferredDeletions, ...langExcluded, ...outOfScope]),
+ carryForwardMtimes(currentMtimes, previousMtimes, [...deferredDeletions, ...failedDeletions, ...langExcluded, ...outOfScope]),
previousMtimes,
- replayedAbsolute,
+ [...replayedAbsolute, ...unsettled],
),
latestRev,
- // Lost parses count as parse errors HERE, whatever they are called
- // elsewhere. A run whose worker pool died resolves every later file as
- // null, and those increment `filesSkippedUnparsed` rather than
- // `parseErrors` -- so without this the run wrote an mtime baseline for
- // tens of thousands of files it never parsed or committed, exited 0, and
- // every later incremental map skipped them as unchanged. Half the graph
- // missing, recoverable only by `--force`, and worse than `main`, which
- // respawns without a cap and loses one file per crash.
- parseErrors + crashedParses(),
- commitErrors,
scopedRun && previousBaseline ? new Date(previousBaseline.lastIngestAt) : undefined,
nextDeletedFiles,
- baselineExtractor,
- replayedFiles,
+ // An unfinished re-ingest must not vouch for the new extractor: files
+ // that failed would then be hash-clean next run and keep the old output.
+ runClean ? baselineExtractor : (previousBaseline?.extractor ?? baselineExtractor),
+ { replayedFiles, pendingFiles, parseTimeouts },
);
if (rebuildProgress !== null) {
// Finished: the baseline now records the new extractor. Otherwise keep
// what landed, so the next run resumes from here.
- if (baselinePersisted) clearRebuildProgress(projectRoot);
+ if (baselinePersisted && runClean) clearRebuildProgress(projectRoot);
else saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress);
}
@@ -3912,6 +3985,7 @@ export async function ingestFiles(
stitchErrors,
stitchSkipped,
stitchSkippedRule,
+ parseTimeouts: timedOutParses(),
graphUnchanged:
opts.force !== true &&
!workspaceMigrated &&
@@ -3960,6 +4034,9 @@ export async function ingestFiles(
// reads as an unexplained regression without the reason.
process.stderr.write(` ${describeStitchSkipped(stitchSkipped, stitchSkippedRule, debug)}\n`);
}
+ if (summary.parseTimeouts.length > 0) {
+ process.stderr.write(` ${describeParseTimeouts(summary.parseTimeouts)}\n`);
+ }
if (replayedChanges.size > 0) {
// stderr and exit 0: nothing failed on this side, and the next run retries.
process.stderr.write(` ${describeReplayedChanges(summary.replayedChanges)}\n`);
@@ -4005,7 +4082,12 @@ export async function ingestFiles(
// `minifiedLikely` and `unparsed` are the buckets that are subsets of
// `filesSkipped`; `parseError` and `tooLarge` are counted separately and
// always were.
- skipReasons: { unchanged: filesSkippedAsUnchanged, emptyFile: filesSkippedAsEmpty, parseError: parseErrors + crashedParses(), unparsed: filesSkippedUnparsed, tooLarge, minifiedLikely, outsideRoot },
+ skipReasons: {
+ unchanged: filesSkippedAsUnchanged, emptyFile: filesSkippedAsEmpty, parseError: parseErrors + crashedParses(),
+ unparsed: filesSkippedUnparsed, parseTimeout: summary.parseTimeouts.length, tooLarge, minifiedLikely, outsideRoot,
+ unreadable: discoveryCounts.unreadable, skippedDirs: discoveryCounts.skippedDirs,
+ },
+ parseTimeouts: summary.parseTimeouts,
commitErrors,
stitchErrors,
// Ix#568. Present only when the stitch was refused before it was sent, so
@@ -4053,9 +4135,12 @@ export async function ingestFiles(
if (replayedChanges.size > 0) console.log(` ${chalk.yellow('not applied:')} ${replayedChanges.size} ${chalk.dim('(see the warning above)')}`);
if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);
if (minifiedLikely > 0) console.log(` ${chalk.dim('skipped minified:')} ${minifiedLikely}`);
+ if (summary.parseTimeouts.length > 0) console.log(` ${chalk.yellow("skipped parse timeout:")} ${summary.parseTimeouts.length} ${chalk.dim(`(${summary.parseTimeouts.join(", ")})`)}`);
// Not dimmed like the others: these were dropped because the repo pointed
// at files outside itself, which is worth a look rather than a shrug.
if (outsideRoot > 0) console.log(` ${chalk.yellow('skipped outside root:')} ${outsideRoot} ${chalk.dim('(symlinks leaving the tree)')}`);
+ if (discoveryCounts.unreadable > 0) console.log(` ${chalk.yellow("skipped unreadable:")} ${discoveryCounts.unreadable} ${chalk.dim("(listed but could not be read)")}`);
+ if (discoveryCounts.skippedDirs > 0) console.log(` ${chalk.dim("skipped build/dependency dirs:")} ${discoveryCounts.skippedDirs}`);
console.log(` rev: ${latestRev}`);
if (patchesApplied === 0 && filesDiscovered === 0) {
diff --git a/ix-cli/src/cli/commands/parse-pool.ts b/ix-cli/src/cli/commands/parse-pool.ts
index 44f5f0b9..529daecf 100644
--- a/ix-cli/src/cli/commands/parse-pool.ts
+++ b/ix-cli/src/cli/commands/parse-pool.ts
@@ -26,8 +26,22 @@ type Task = {
resolve: (result: unknown) => void;
/** Does losing this one mean the RUN lost a file? See `parse`. */
counts: boolean;
+ /** The backstop for a parse that never answers; see `onTaskTimeout`. */
+ timer?: ReturnType<typeof setTimeout>;
};
+/**
+ * The per-file parse budget the worker enforces, read as core-ingestion's
+ * `parseBudgetMs()` reads it: IX_PARSE_BUDGET_MS, default 10 s, 0 = none.
+ * Restated rather than imported, because ix-cli does not depend on that
+ * package at runtime.
+ */
+function parseBudgetMsFromEnv(): number {
+ const raw = process.env.IX_PARSE_BUDGET_MS;
+ const n = raw === undefined || raw.trim() === '' ? Number.NaN : Number(raw);
+ return Number.isFinite(n) && n >= 0 ? n : 10_000;
+}
+
export class ParsePool {
private workers: Worker[] = [];
private idle: Worker[] = [];
@@ -39,21 +53,47 @@ export class ParsePool {
* stops waiting for it.
* @param maxWaitMs The ceiling on waiting for a BUSY one.
*
- * Both injectable only so the tests can pin the behaviour without spending
- * the real periods -- the ceiling test would otherwise cost ten seconds on
- * every leg of the matrix. `ingestFiles` passes neither.
+ * @param taskTimeoutMs How long one parse may hold a worker before the pool
+ * gives up on it; 0 = never. See `onTaskTimeout`.
+ *
+ * All three injectable only so the tests can pin the behaviour without
+ * spending the real periods -- the ceiling test would otherwise cost ten
+ * seconds on every leg of the matrix. `ingestFiles` passes none of them.
*/
constructor(
private workerPath: string,
private concurrency: number,
private graceMs: number = ParsePool.SHUTDOWN_GRACE_MS,
private maxWaitMs: number = ParsePool.SHUTDOWN_MAX_WAIT_MS,
+ private taskTimeoutMs: number = 2 * parseBudgetMsFromEnv(),
) {}
+ /**
+ * Start every worker now. Optional: without it the pool starts workers as
+ * work queues up (see `drain`), which is what `ingestFiles` wants -- a
+ * one-file edit then starts one worker, not `concurrency` of them.
+ */
init(): void {
- for (let i = 0; i < this.concurrency; i++) {
- this.spawnWorker();
- }
+ while (this.started < this.concurrency) this.startWorker();
+ }
+
+ /**
+ * Raise the ceiling on workers. A later, larger phase of a run asks for more
+ * than the first one did; the pool never shrinks, and the new workers start
+ * only if queued work needs them.
+ */
+ growTo(concurrency: number): void {
+ if (concurrency <= this.concurrency) return;
+ this.concurrency = concurrency;
+ this.drain();
+ }
+
+ /** Workers started to fill the ceiling. Replacements for dead ones are not counted; see `onError`. */
+ private started = 0;
+
+ private startWorker(): void {
+ this.started++;
+ this.spawnWorker();
}
/**
@@ -132,7 +172,10 @@ export class ParsePool {
// Not counted, for the same reason the stranded queue is not: every gate
// that reads `crashedParses()` has already run by the time `destroy()` is
// called from the outermost `finally`.
- for (const task of this.active.values()) task.resolve(null);
+ for (const task of this.active.values()) {
+ clearTimeout(task.timer);
+ task.resolve(null);
+ }
this.active.clear();
this.workers = [];
this.idle = [];
@@ -421,12 +464,77 @@ export class ParsePool {
}
private drain(): void {
+ // Start workers on demand, one per queued task no idle worker can take,
+ // up to the ceiling. Only first starts: a dead worker's replacement is
+ // `onError`'s, under the respawn cap, so a worker that dies on load cannot
+ // turn this into a spawn loop.
+ while (!this.dead && !this.destroyed && this.queue.length > this.idle.length && this.started < this.concurrency) {
+ this.startWorker();
+ }
while (this.idle.length > 0 && this.queue.length > 0) {
const w = this.idle.pop()!;
const task = this.queue.shift()!;
this.active.set(w, task);
w.postMessage({ filePath: task.filePath, source: task.source });
+ if (this.taskTimeoutMs > 0) {
+ task.timer = setTimeout(() => this.onTaskTimeout(w, task), this.taskTimeoutMs);
+ // A pending backstop must not hold the process open on its own.
+ task.timer.unref?.();
+ }
+ }
+ }
+
+ /**
+ * Files whose parse ran past the budget, in the order they gave up --
+ * reported by the worker's own budget or by the backstop below. Only tasks
+ * that count (see `parse`). Not counted in `crashedTasks()`: a crash leaves
+ * every file the run could not parse unsettled in the baseline, and one slow
+ * file must not do that to the rest. The ingest keeps just these files
+ * unsettled and records them for `ix status`.
+ */
+ timedOutFiles(): string[] {
+ return [...this.timedOut];
+ }
+
+ private timedOut: string[] = [];
+
+ /**
+ * The backstop for a parse that never answers.
+ *
+ * The worker enforces the parse budget itself, but only between matches: a
+ * file stuck inside tree-sitter's native parse or query never reaches a
+ * check. So past twice the budget the pool settles the task as a timeout,
+ * lets the worker go and starts a replacement under the same respawn cap a
+ * crash uses.
+ *
+ * Let go, NOT terminated -- `shutdown` explains at length why `terminate()`
+ * segfaults the process. The thread is asked to close once it is done and is
+ * `unref`'d meanwhile, so it burns its core until the native call returns or
+ * the process exits, and its late answer goes nowhere.
+ */
+ private onTaskTimeout(w: Worker, task: Task): void {
+ if (this.destroyed || this.active.get(w) !== task) return;
+ this.active.delete(w);
+ if (task.counts) this.timedOut.push(task.filePath);
+ task.resolve(null);
+ const idx = this.workers.indexOf(w);
+ if (idx !== -1) this.workers.splice(idx, 1);
+ // As `shutdown` does when it gives up, and for the same reasons: drop the
+ // listeners that close over the pool, keep one that captures nothing.
+ w.removeAllListeners('message');
+ w.removeAllListeners('exit');
+ w.removeAllListeners('error');
+ w.on('error', SWALLOW_ERROR);
+ w.postMessage({ __shutdown: true });
+ w.unref();
+ if (this.respawns < ParsePool.MAX_RESPAWNS) {
+ this.respawns++;
+ this.spawnWorker();
+ } else if (this.workers.length === 0) {
+ this.latchDead();
+ return;
}
+ this.drain();
}
/**
@@ -513,10 +621,12 @@ export class ParsePool {
return this.deaths;
}
- private onResult(w: Worker, msg: { ok: boolean; result: unknown }): void {
+ private onResult(w: Worker, msg: { ok: boolean; result: unknown; reason?: string }): void {
const task = this.active.get(w);
if (!task) return;
this.active.delete(w);
+ clearTimeout(task.timer);
+ if (!msg.ok && msg.reason === 'timeout' && task.counts) this.timedOut.push(task.filePath);
// A successful round trip clears the respawn budget. The cap exists to
// stop a worker that dies deterministically from spinning spawn -> die ->
// spawn; it is not meant to be a lifetime quota. As a per-run total it made
@@ -579,6 +689,7 @@ export class ParsePool {
const task = this.active.get(w);
if (task) {
this.active.delete(w);
+ clearTimeout(task.timer);
if (task.counts) this.crashed++;
task.resolve(null); // isolate: failed file = null parse result
}
@@ -627,19 +738,20 @@ export class ParsePool {
// none, the `else if` latches `dead`, strands the queue and resolves it
// as crashed, then returns -- so that path never reaches `drain()`. With
// others still alive, NEITHER branch body runs and control falls straight
- // through to `drain()`. `ingest.ts` builds the pool with
- // `Math.max(1, os.cpus().length - 1)` and `respawns` is a pool-wide budget
- // that any success resets, so on an ordinary machine the fall-through is
- // the usual case -- do not read the cap as implying the pool is finished.
+ // through to `drain()`. `ingest.ts` sizes the pool with `parsePoolSize`
+ // -- one worker per 50 files, at most the cores less one or 8 -- and
+ // `respawns` is a pool-wide budget that any success resets, so on a run
+ // of any size the fall-through is the usual case -- do not read the cap as
+ // implying the pool is finished.
//
- // The floor in that expression is there because 1 is reachable, and at
- // concurrency 1 this inverts: the first death PAST the cap is also the last
+ // Concurrency 1 is common -- any run of 50 files or fewer -- and there
+ // this inverts: the first death PAST the cap is also the last
// worker, so `workers.length === 0` always holds and the pool always
// latches `dead`. (Not the death that exhausts the budget -- that one
// still takes the respawn branch and spawns a replacement. The two are
// one apart, and this file is where that distinction has to stay
- // straight.) Above concurrency 1 it does not invert. Which case a given machine falls
- // into depends on its core count, and this comment deliberately does not
+ // straight.) Above concurrency 1 it does not invert. Which case a given run falls
+ // into depends on its size and the core count, and this comment deliberately does not
// say -- an earlier revision guessed at CI's and contradicted the machine
// sizes in `docs/parse-pool-teardown.md`, which is the file that owns
// them.
@@ -649,18 +761,24 @@ export class ParsePool {
this.respawns++;
this.spawnWorker();
} else if (this.workers.length === 0) {
- // Out of workers and out of replacements. Nothing queued can ever be
- // parsed, so resolve it rather than leave `Promise.all` waiting on a pool
- // that no longer exists -- and LATCH it, because `parse()` must answer the
- // same way for every call after this, not just for what happened to be
- // queued at this instant. Counted as crashed, which is what it is, so the
- // stitch gate knows this run lost files.
- this.dead = true;
- const stranded = this.queue.splice(0, this.queue.length);
- this.crashed += stranded.filter(t => t.counts).length;
- for (const t of stranded) t.resolve(null);
+ this.latchDead();
return;
}
this.drain();
}
+
+ /**
+ * Out of workers and out of replacements. Nothing queued can ever be
+ * parsed, so resolve it rather than leave `Promise.all` waiting on a pool
+ * that no longer exists -- and LATCH it, because `parse()` must answer the
+ * same way for every call after this, not just for what happened to be
+ * queued at this instant. Counted as crashed, which is what it is, so the
+ * stitch gate knows this run lost files.
+ */
+ private latchDead(): void {
+ this.dead = true;
+ const stranded = this.queue.splice(0, this.queue.length);
+ this.crashed += stranded.filter(t => t.counts).length;
+ for (const t of stranded) t.resolve(null);
+ }
}
diff --git a/ix-cli/src/cli/commands/status.ts b/ix-cli/src/cli/commands/status.ts
index e3f6002d..bcd5eeeb 100644
--- a/ix-cli/src/cli/commands/status.ts
+++ b/ix-cli/src/cli/commands/status.ts
@@ -20,6 +20,8 @@ interface StatusStaleInfo {
sampleChangedFiles: string[];
/** Changed files the last run could not get applied (F-01); see `IngestBaseline.replayedFiles`. */
replayedFiles?: string[];
+ /** Files skipped because their parse ran past the budget; see `IngestBaseline.parseTimeouts`. */
+ parseTimeouts?: string[];
}
/**
@@ -47,8 +49,10 @@ export function renderStatusLlm(
["last_ingest_at", staleInfo?.lastIngestAt ?? null],
["stale_files", staleInfo ? String(staleInfo.staleFiles) : null],
["not_applied", staleInfo ? String(staleInfo.replayedFiles?.length ?? 0) : null],
+ ["parse_timeouts", staleInfo ? String(staleInfo.parseTimeouts?.length ?? 0) : null],
["stale", staleInfo
- ? (!staleInfo.graphCompleted || staleInfo.staleFiles > 0 || (staleInfo.replayedFiles?.length ?? 0) > 0 ? "true" : "false")
+ ? (!staleInfo.graphCompleted || staleInfo.staleFiles > 0 || (staleInfo.replayedFiles?.length ?? 0) > 0
+ || (staleInfo.parseTimeouts?.length ?? 0) > 0 ? "true" : "false")
: null],
])];
for (const f of staleInfo?.sampleChangedFiles ?? []) {
@@ -57,9 +61,20 @@ export function renderStatusLlm(
for (const f of staleInfo?.replayedFiles ?? []) {
lines.push(llmLine("not_applied", [["path", f]]));
}
+ for (const f of staleInfo?.parseTimeouts ?? []) {
+ lines.push(llmLine("parse_timeout", [["path", f]]));
+ }
return lines;
}
+/** The `ix status` warning for files the last run skipped on the parse budget. */
+function describeStatusParseTimeouts(files: readonly string[], sample = 5): string {
+ const shown = files.slice(0, sample).join(", ");
+ const more = files.length > sample ? ` and ${files.length - sample} more` : "";
+ return `${files.length} file(s) are not in the graph: their parse ran past the per-file budget (${shown}${more}). ` +
+ "Raise IX_PARSE_BUDGET_MS (milliseconds, 0 = none) and run ix map.";
+}
+
export function registerStatusCommand(program: Command): void {
program
.command("status")
@@ -92,6 +107,7 @@ export function registerStatusCommand(program: Command): void {
staleFiles: staleInfo?.staleFiles ?? 0,
sampleChangedFiles: staleInfo?.sampleChangedFiles ?? [],
replayedFiles: staleInfo?.replayedFiles ?? [],
+ parseTimeouts: staleInfo?.parseTimeouts ?? [],
};
printJson(result);
} else {
@@ -120,9 +136,14 @@ export function registerStatusCommand(program: Command): void {
renderNote(`... and ${staleInfo.staleFiles - staleInfo.sampleChangedFiles.length} more`);
}
renderNote("Run ix map to update.");
- } else {
+ } else if (staleInfo.parseTimeouts.length === 0) {
renderSuccess("Graph is up to date.");
}
+ if (staleInfo.graphCompleted && staleInfo.parseTimeouts.length > 0) {
+ // Beside the other states, not instead of them: another map
+ // will not bring these files in, a larger budget will.
+ renderWarning(describeStatusParseTimeouts(staleInfo.parseTimeouts));
+ }
if (staleInfo.graphCompleted && !staleInfo.mapCompleted) {
renderWarning("No completed architecture map is recorded for this source revision.");
renderNote("Source graph reads remain available, but hierarchy views may be incomplete.");
diff --git a/ix-cli/src/cli/commands/watch.ts b/ix-cli/src/cli/commands/watch.ts
index ae1a7ea4..6ef60790 100644
--- a/ix-cli/src/cli/commands/watch.ts
+++ b/ix-cli/src/cli/commands/watch.ts
@@ -8,19 +8,8 @@ import type { Command } from "commander";
import chalk from "chalk";
import { resolveWorkspaceRoot, clearIngestMtimeCache } from "../config.js";
import { bootstrap, ensureWorkspaceIdState } from "../bootstrap.js";
-import { SUPPORTED_EXTENSIONS } from "../supported-extensions.js";
+import { discoverSourceFiles, isGeneratedFile, isSupportedSourceFile, SKIPPED_DIRS } from "../file-discovery.js";
-const SUPPORTED_NAMES = new Set([
- ".gitignore", ".gitattributes", ".editorconfig", ".env",
- ".eslintrc", ".prettierrc", ".babelrc",
- "Makefile", "Dockerfile", "Procfile", "Gemfile", "Rakefile",
- "BUILD", "WORKSPACE",
-]);
-
-const IGNORE_DIRS = new Set([
- "node_modules", ".git", "dist", "build", "target", ".next",
- ".cache", "__pycache__", ".ix", ".claude",
-]);
const DEBOUNCE_MS = 300;
const MAP_COALESCED_EXIT_CODE = 75;
@@ -216,15 +205,15 @@ export function updatePollingSnapshot(
return changed;
}
+/** The files `ix map` would ingest -- the same test discovery applies. */
function isSupportedPath(filePath: string): boolean {
- const ext = path.extname(filePath).toLowerCase();
- return SUPPORTED_EXTENSIONS.has(ext) || SUPPORTED_NAMES.has(path.basename(filePath));
+ return isSupportedSourceFile(filePath) && !isGeneratedFile(path.basename(filePath));
}
export function shouldWatch(root: string, filePath: string): boolean {
if (!isSupportedPath(filePath)) return false;
const segments = path.relative(root, filePath).split(path.sep);
- return !segments.some(segment => IGNORE_DIRS.has(segment));
+ return !segments.some(segment => SKIPPED_DIRS.has(segment));
}
export function prepareMigratedWorkspaceRefresh(
@@ -234,23 +223,9 @@ export function prepareMigratedWorkspaceRefresh(
clear(root);
}
+/** The files `ix map` would discover under `dir` (see `file-discovery.ts`). */
function collectFiles(dir: string): string[] {
- const results: string[] = [];
- const stack = [dir];
- while (stack.length > 0) {
- const current = stack.pop()!;
- let entries: fs.Dirent[];
- try { entries = fs.readdirSync(current, { withFileTypes: true }); }
- catch { continue; }
- for (const entry of entries) {
- if (IGNORE_DIRS.has(entry.name)) continue;
- if (entry.name.startsWith(".") && entry.isDirectory()) continue;
- const fullPath = path.join(current, entry.name);
- if (entry.isDirectory()) stack.push(fullPath);
- else if (entry.isFile() && isSupportedPath(fullPath)) results.push(fullPath);
- }
- }
- return results;
+ return discoverSourceFiles(dir);
}
function createBatchNotifier(root: string, scheduler: WatchRefreshScheduler): {
diff --git a/ix-cli/src/cli/file-discovery.ts b/ix-cli/src/cli/file-discovery.ts
new file mode 100644
index 00000000..2e4dccb1
--- /dev/null
+++ b/ix-cli/src/cli/file-discovery.ts
@@ -0,0 +1,303 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import * as fs from "node:fs";
+import * as nodePath from "node:path";
+import { spawnSync } from "node:child_process";
+
+import { createIgnoreMatcher, parseIgnoreFile, type IgnoreMatcher } from "./ignore-globs.js";
+import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";
+
+/**
+ * Which files are the repository: one implementation for `ix ingest`/`ix map`,
+ * `ix watch` and `ix status`.
+ *
+ * There were three, and they disagreed. The filesystem walk skipped `tests/`,
+ * `examples/`, `fixtures/`, Go's `pkg/` and every dot-directory, while
+ * `git ls-files` -- the path in any git repository -- skipped none of them, so
+ * the same tree gave a different graph depending on whether it was a git
+ * checkout. `watch` and `status` had their own lists again. And `git ls-files`
+ * quoted any non-ASCII name (`"caf\303\251.ts"`), the stat on that string
+ * failed, and the file was silently dropped.
+ *
+ * Now: git lists the files when it can (NUL-separated, unquoted); otherwise the
+ * walk honours each directory's `.gitignore`, so the two agree on an ordinary
+ * tree. The walk also skips a short list of directories that are never source
+ * (build output, dependencies, VCS metadata, caches) by name. In a work tree
+ * git's own ignore rules decide instead: a tracked file is kept whatever its
+ * directory is called -- a committed `build/` or `out/` can be source -- and
+ * only untracked files under those names are skipped. Both count what they
+ * skipped. Tests and fixtures are in the set; the role classifier is what
+ * tells them apart.
+ *
+ * The walk's `.gitignore` reading is the `.ixignore` subset (see
+ * `ignore-globs.ts`): no `!` negation, so it can exclude a little more than
+ * git does.
+ */
+
+/**
+ * Never source: build output, dependencies, VCS metadata, tool caches. Skipped
+ * by name in a walk, and for untracked files in a work tree; never for a file
+ * git tracks.
+ */
+export const SKIPPED_DIRS: ReadonlySet<string> = new Set([
+ "node_modules", ".git", ".hg", ".svn", "dist", "build", "target", "out", ".next",
+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
+ ".gradle", ".idea", ".vscode", ".settings", ".vs",
+ ".cache", ".parcel-cache", "coverage", ".nyc_output",
+ "Pods", ".dart_tool", ".pub-cache", "obj",
+ ".terraform", ".serverless", ".turbo", ".svelte-kit", ".nuxt", ".angular", ".expo",
+ ".ix", ".claude", ".gitnexus",
+]);
+
+// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.
+// These files inflate parse time and graph size without contributing to architectural signal.
+const GENERATED_FILE_SUFFIXES = [
+ ".pb.go", // protobuf generated Go
+ "_deepcopy.go", // controller-gen deepcopy (Kubernetes ecosystem)
+ "_mock.go", // gomock generated mocks
+ ".pb.ts", // protobuf generated TypeScript
+ ".pb.js", // protobuf generated JavaScript
+ "_pb.ts", // alternative protobuf TypeScript naming
+ "_pb.js", // alternative protobuf JavaScript naming
+];
+
+const GENERATED_FILE_PREFIXES = [
+ "zz_generated", // controller-gen convention (Kubernetes ecosystem)
+ "mock_", // gomock convention
+];
+
+// Generated dependency lockfiles, matched whole. Only the ones whose extension
+// SUPPORTED_EXTENSIONS actually discovers are listed: `yarn.lock`,
+// `composer.lock`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock` and friends end
+// in `.lock`, which is not a discovered extension, so they never reach here.
+// A lockfile well under MAX_FILE_BYTES still expands into a graph patch larger
+// than a proxy will accept — 646 KB was enough for an HTTP 413 (Ix#523) — and
+// it describes a resolved dependency snapshot, not architecture.
+const GENERATED_FILE_NAMES = new Set([
+ "package-lock.json", // npm
+ "npm-shrinkwrap.json", // npm, published form
+ "pnpm-lock.yaml", // pnpm
+ "packages.lock.json", // NuGet
+]);
+
+export function isGeneratedFile(basename: string): boolean {
+ // Lowercased for the name match alone: `isSupportedSourceFile` lowercases
+ // too, so on a case-insensitive filesystem a `Package-lock.json` is
+ // discovered and would otherwise slip past a case-sensitive comparison.
+ if (GENERATED_FILE_NAMES.has(basename.toLowerCase())) return true;
+ for (const suffix of GENERATED_FILE_SUFFIXES) {
+ if (basename.endsWith(suffix)) return true;
+ }
+ for (const prefix of GENERATED_FILE_PREFIXES) {
+ if (basename.startsWith(prefix)) return true;
+ }
+ return false;
+}
+
+export function isSupportedSourceFile(filePath: string): boolean {
+ const fileName = nodePath.basename(filePath).toLowerCase();
+ return fileName === "dockerfile"
+ || fileName.endsWith(".dockerfile")
+ || fileName === "makefile"
+ || fileName === "gnumakefile"
+ || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
+}
+
+/** A file discovery would hand to the parser. */
+function isCandidate(filePath: string): boolean {
+ return isSupportedSourceFile(filePath) && !isGeneratedFile(nodePath.basename(filePath));
+}
+
+export type DiscoveryExclude = { matcher: IgnoreMatcher; root: string; onSkip?: () => void };
+
+/** What discovery left out, so it can be reported rather than guessed at. */
+export interface DiscoveryCounts {
+ /** Untracked files under a `SKIPPED_DIRS` directory (git mode) or such directories (walk). */
+ skippedDirs: number;
+ /** Listed files that could not be stat'd: permissions, a race with a delete. */
+ unreadable: number;
+}
+
+export function emptyDiscoveryCounts(): DiscoveryCounts {
+ return { skippedDirs: 0, unreadable: 0 };
+}
+
+export function canonicalizeDiscoveredFilePath(filePath: string): string {
+ try {
+ return fs.realpathSync.native(filePath);
+ } catch {
+ return nodePath.resolve(filePath);
+ }
+}
+
+function inSkippedDir(relativePath: string): boolean {
+ const parts = relativePath.split(/[/\\]/);
+ parts.pop();
+ return parts.some(p => SKIPPED_DIRS.has(p));
+}
+
+const GIT_SAFE_CONFIG = [
+ "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.quotePath=false",
+];
+
+/**
+ * The files `git ls-files` reports under `dir`, canonical and de-duplicated, or
+ * null when `dir` is not in a git work tree.
+ *
+ * `-z` and `core.quotePath=false`: without them git quotes a non-ASCII name
+ * (`"caf\303\251.ts"`), the stat on that string fails, and the file vanished.
+ */
+export function tryGitLsFiles(
+ dir: string,
+ recursive: boolean,
+ exclude?: DiscoveryExclude,
+ counts: DiscoveryCounts = emptyDiscoveryCounts(),
+): string[] | null {
+ try {
+ const result = spawnSync(
+ "git",
+ // core.fsmonitor / core.hooksPath: a cloned repository's own .git/config
+ // can name a command ls-files would run. Never let it.
+ [...GIT_SAFE_CONFIG, "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
+ {
+ cwd: dir,
+ encoding: "utf-8",
+ maxBuffer: 64 * 1024 * 1024,
+ },
+ );
+ if (result.status !== 0) return null;
+
+ // Asked only when a listed file sits under a skipped name.
+ let untracked: Set<string> | null = null;
+ const isUntracked = (entry: string): boolean => {
+ untracked ??= gitUntracked(dir);
+ return untracked.has(entry);
+ };
+ const files: string[] = [];
+ const seenCanonical = new Set<string>();
+ for (const entry of result.stdout.split("\0")) {
+ if (!entry) continue;
+ const fullPath = nodePath.resolve(dir, entry);
+ if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
+ // Before the extension check, so the count reports what the caller
+ // excluded rather than what they excluded and Ix would have skipped
+ // anyway.
+ if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, fullPath))) {
+ exclude.onSkip?.();
+ continue;
+ }
+ if (!isCandidate(fullPath)) continue;
+ // A tracked file stays: git is told what is generated through
+ // .gitignore, and a repository that commits a `build/` or `out/` tree
+ // is saying it is source. An untracked one under such a name is what a
+ // missing .gitignore line would have caught -- a fresh `node_modules`.
+ if (inSkippedDir(entry) && isUntracked(entry)) { counts.skippedDirs++; continue; }
+ try {
+ if (!fs.statSync(fullPath).isFile()) continue;
+ } catch {
+ counts.unreadable++;
+ continue;
+ }
+ const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);
+ if (seenCanonical.has(canonicalPath)) continue;
+ seenCanonical.add(canonicalPath);
+ files.push(canonicalPath);
+ }
+ return files;
+ } catch {
+ return null;
+ }
+}
+
+/** Untracked, not-ignored entries under `dir`, as `git ls-files` spells them. */
+function gitUntracked(dir: string): Set<string> {
+ const result = spawnSync("git", [...GIT_SAFE_CONFIG, "ls-files", "-z", "--others", "--exclude-standard"], {
+ cwd: dir,
+ encoding: "utf-8",
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ // Unknown: treat every listed file as tracked, which keeps it.
+ if (result.status !== 0) return new Set();
+ return new Set(result.stdout.split("\0").filter(Boolean));
+}
+
+/** `.gitignore` matchers in force for a directory: each one's own, with the directory it is relative to. */
+type GitignoreStack = Array<{ dir: string; matcher: IgnoreMatcher }>;
+
+function withGitignore(dir: string, stack: GitignoreStack): GitignoreStack {
+ try {
+ const patterns = parseIgnoreFile(fs.readFileSync(nodePath.join(dir, ".gitignore"), "utf-8"));
+ if (patterns.length > 0) return [...stack, { dir, matcher: createIgnoreMatcher(patterns) }];
+ } catch {
+ // No .gitignore here, the ordinary case.
+ }
+ return stack;
+}
+
+function gitignored(stack: GitignoreStack, fullPath: string, isDirectory: boolean): boolean {
+ return stack.some(({ dir, matcher }) => matcher.matches(nodePath.relative(dir, fullPath), isDirectory));
+}
+
+/**
+ * Walk `dir` for candidate files, honouring every `.gitignore` on the way down
+ * and skipping `SKIPPED_DIRS`. Yields paths as found; `discoverSourceFiles`
+ * canonicalizes them.
+ */
+export function* walkFiles(
+ dir: string,
+ recursive: boolean,
+ exclude?: DiscoveryExclude,
+ counts: DiscoveryCounts = emptyDiscoveryCounts(),
+ gitignores: GitignoreStack = [],
+): Generator<string> {
+ let entries: fs.Dirent[];
+ try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
+ catch { return; }
+ const stack = withGitignore(dir, gitignores);
+
+ for (const entry of entries) {
+ const isDirectory = entry.isDirectory();
+ if (isDirectory && SKIPPED_DIRS.has(entry.name)) { counts.skippedDirs++; continue; }
+ const full = nodePath.join(dir, entry.name);
+ // Directories are tested before descending, so an excluded tree costs one
+ // check rather than one per file inside it.
+ if (exclude && exclude.matcher.matches(nodePath.relative(exclude.root, full), isDirectory)) {
+ exclude.onSkip?.();
+ continue;
+ }
+ if (gitignored(stack, full, isDirectory)) continue;
+ if (isDirectory) {
+ if (recursive) yield* walkFiles(full, true, exclude, counts, stack);
+ } else if (entry.isFile()) {
+ if (isCandidate(entry.name)) yield full;
+ }
+ }
+}
+
+/**
+ * The source files under `dir`, canonical and de-duplicated: git's list when
+ * `dir` is in a work tree, else the walk.
+ *
+ * `walkLimit` stops the walk after that many files. A git listing is bounded
+ * by the repository, but a walk is bounded by nothing: a command run from a
+ * home directory would read the whole disk.
+ */
+export function discoverSourceFiles(
+ dir: string,
+ opts: { recursive?: boolean; exclude?: DiscoveryExclude; counts?: DiscoveryCounts; walkLimit?: number } = {},
+): string[] {
+ const recursive = opts.recursive ?? true;
+ const counts = opts.counts ?? emptyDiscoveryCounts();
+ const listed = tryGitLsFiles(dir, recursive, opts.exclude, counts);
+ if (listed) return listed;
+ const out: string[] = [];
+ const seen = new Set<string>();
+ for (const filePath of walkFiles(dir, recursive, opts.exclude, counts)) {
+ if (opts.walkLimit !== undefined && out.length >= opts.walkLimit) break;
+ const canonical = canonicalizeDiscoveredFilePath(filePath);
+ if (seen.has(canonical)) continue;
+ seen.add(canonical);
+ out.push(canonical);
+ }
+ return out;
+}
diff --git a/ix-cli/src/cli/ingest-baseline.ts b/ix-cli/src/cli/ingest-baseline.ts
index d44e4d8a..df72df96 100644
--- a/ix-cli/src/cli/ingest-baseline.ts
+++ b/ix-cli/src/cli/ingest-baseline.ts
@@ -13,6 +13,8 @@ interface SerializedIngestBaseline {
tracksMapBaseline?: boolean;
extractor?: string;
replayedFiles?: string[];
+ pendingFiles?: string[];
+ parseTimeouts?: string[];
}
export interface IngestBaseline {
@@ -46,8 +48,40 @@ export interface IngestBaseline {
* are kept at the previous value, so the next run sends them again.
*/
replayedFiles: string[];
+ /**
+ * Workspace-relative paths of changed files the last run could not ingest:
+ * a read or build error, a failed commit, a parse lost to a dead worker.
+ * Each keeps its previous mtime, so the next run retries it. A new file has
+ * no previous mtime and is left out of `files`, and a file missing from
+ * `files` is judged stale against `lastIngestAt`, which that same run moved
+ * past the file's mtime -- so without this list `ix status` called the
+ * graph current while the file was not in it.
+ */
+ pendingFiles: string[];
+ /**
+ * Workspace-relative paths of files whose parse ran past the per-file
+ * budget (`IX_PARSE_BUDGET_MS`) on the last run, so they are not in the
+ * graph. Not settled: like `pendingFiles` they keep their previous mtime
+ * (or none) and every run tries them again, and `ix status` warns about
+ * them rather than calling the graph current. A slow file times out on
+ * every run, so recording it clean skipped it for good.
+ */
+ parseTimeouts: string[];
+}
+
+/** Per-file lists a baseline write records beside the mtimes. */
+export interface BaselineFileNotes {
+ /** See `IngestBaseline.replayedFiles`. */
+ replayedFiles?: readonly string[];
+ /** See `IngestBaseline.pendingFiles`. */
+ pendingFiles?: readonly string[];
+ /** See `IngestBaseline.parseTimeouts`. */
+ parseTimeouts?: readonly string[];
}
+const stringList = (value: unknown): string[] =>
+ Array.isArray(value) ? value.filter((p): p is string => typeof p === "string") : [];
+
/**
* What counts as a revision, for both sides of this file.
*
@@ -93,9 +127,9 @@ export function loadIngestBaseline(projectRoot: string): IngestBaseline | null {
lastIngestAt,
tracksMapBaseline: data.tracksMapBaseline === true,
extractor: typeof data.extractor === "string" ? data.extractor : null,
- replayedFiles: Array.isArray(data.replayedFiles)
- ? data.replayedFiles.filter((p): p is string => typeof p === "string")
- : [],
+ replayedFiles: stringList(data.replayedFiles),
+ pendingFiles: stringList(data.pendingFiles),
+ parseTimeouts: stringList(data.parseTimeouts),
};
} catch {
return null;
@@ -109,7 +143,7 @@ export function saveIngestBaseline(
now: Date = new Date(),
deletedFiles: Map<string, string[]> = new Map(),
extractor?: string | null,
- replayedFiles: readonly string[] = [],
+ { replayedFiles = [], pendingFiles = [], parseTimeouts = [] }: BaselineFileNotes = {},
): void {
try {
// Keep the last good rev rather than writing a shape the read side will
@@ -134,9 +168,23 @@ export function saveIngestBaseline(
tracksMapBaseline: true,
...(extractor ? { extractor } : {}),
...(replayedFiles.length > 0 ? { replayedFiles: [...replayedFiles].sort() } : {}),
+ ...(pendingFiles.length > 0 ? { pendingFiles: [...new Set(pendingFiles)].sort() } : {}),
+ ...(parseTimeouts.length > 0 ? { parseTimeouts: [...new Set(parseTimeouts)].sort() } : {}),
};
- fs.mkdirSync(path.dirname(ingestMtimeCachePath(projectRoot)), { recursive: true });
- fs.writeFileSync(ingestMtimeCachePath(projectRoot), JSON.stringify(data));
+ const target = ingestMtimeCachePath(projectRoot);
+ fs.mkdirSync(path.dirname(target), { recursive: true });
+ // Via a temp file and rename: a process killed mid-write left a truncated
+ // baseline, which reads as none, and the next map re-ingested everything.
+ const tmp = `${target}.${process.pid}.tmp`;
+ try {
+ fs.writeFileSync(tmp, JSON.stringify(data));
+ fs.renameSync(tmp, target);
+ } catch (err) {
+ // A failed rename (a locked target on Windows, say) must not leave the
+ // temp file behind; one would pile up per failing run.
+ fs.rmSync(tmp, { force: true });
+ throw err;
+ }
} catch {
// The cache is an optimization and freshness hint. Ingestion itself succeeded.
}
diff --git a/ix-cli/src/cli/stale.ts b/ix-cli/src/cli/stale.ts
index c787efaf..3422dd16 100644
--- a/ix-cli/src/cli/stale.ts
+++ b/ix-cli/src/cli/stale.ts
@@ -5,7 +5,7 @@ import * as path from "node:path";
import { resolveWorkspaceRoot } from "./config.js";
import { loadIngestBaseline } from "./ingest-baseline.js";
import { hasCompletedMapFor } from "./map-baseline.js";
-import { SUPPORTED_EXTENSIONS } from "./supported-extensions.js";
+import { canonicalizeDiscoveredFilePath, discoverSourceFiles } from "./file-discovery.js";
export interface StaleInfo {
graphCompleted: boolean;
@@ -16,53 +16,30 @@ export interface StaleInfo {
sampleChangedFiles: string[];
/** Changed files the last run could not get applied (F-01); see `IngestBaseline.replayedFiles`. */
replayedFiles: string[];
+ /**
+ * Files the last run skipped because their parse ran past the budget, still
+ * on disk; see `IngestBaseline.parseTimeouts`. Not counted in `staleFiles`:
+ * another run will not fix them, a larger IX_PARSE_BUDGET_MS will.
+ */
+ parseTimeouts: string[];
}
-const SUPPORTED_NAMES = new Set([
- ".gitignore", ".gitattributes", ".editorconfig", ".env",
- ".eslintrc", ".prettierrc", ".babelrc",
- "Makefile", "Dockerfile", "Procfile", "Gemfile", "Rakefile",
- "BUILD", "WORKSPACE",
-]);
-
-const IGNORE_DIRS = new Set([
- "node_modules", ".git", "dist", "build", "target", ".next",
- ".cache", "__pycache__", ".ix", ".claude",
-]);
-
/**
- * Walk a directory and collect file paths with supported extensions.
- * Bounded to prevent runaway on huge repos.
+ * Most files `ix status` reads when the workspace is not a git work tree; the
+ * cap the old status walk had. A walk has no bound of its own, and status runs
+ * on every agent turn: from a home directory it would read the whole disk.
+ * Files past the cap are not checked for changes. They do not look deleted:
+ * a baseline entry counts as deleted only when the file is gone from disk.
*/
-function collectFiles(dir: string, limit: number = 5000): string[] {
- const results: string[] = [];
- const stack = [dir];
+const STATUS_WALK_LIMIT = 5000;
- while (stack.length > 0 && results.length < limit) {
- const current = stack.pop()!;
- let entries: fs.Dirent[];
- try {
- entries = fs.readdirSync(current, { withFileTypes: true });
- } catch {
- continue;
- }
- for (const entry of entries) {
- if (results.length >= limit) break;
- if (entry.name.startsWith(".") && entry.isDirectory()) continue;
- if (IGNORE_DIRS.has(entry.name)) continue;
-
- const fullPath = path.join(current, entry.name);
- if (entry.isDirectory()) {
- stack.push(fullPath);
- } else if (entry.isFile()) {
- const ext = path.extname(entry.name).toLowerCase();
- if (SUPPORTED_EXTENSIONS.has(ext) || SUPPORTED_NAMES.has(entry.name)) {
- results.push(fullPath);
- }
- }
- }
- }
- return results;
+/**
+ * The files `ix map` would discover (see `file-discovery.ts`), so "changed
+ * since the last ingest" is judged over the same set the ingest recorded. A
+ * git listing is complete; a walk stops at `walkLimit`.
+ */
+function collectFiles(dir: string, walkLimit: number): string[] {
+ return discoverSourceFiles(dir, { walkLimit });
}
function differsFromIngestBaseline(
@@ -77,6 +54,16 @@ function differsFromIngestBaseline(
: ingestedMtime !== mtimeMs;
}
+/**
+ * Absolute paths for a baseline's list of files the last run could not
+ * ingest. They count as changed whatever their mtime: a new one is absent from
+ * `files` with an mtime older than `lastIngestAt`, which would otherwise read
+ * as current.
+ */
+function absoluteSet(workspaceRoot: string, relPaths: readonly string[]): Set<string> {
+ return new Set(relPaths.map((rel) => path.resolve(workspaceRoot, rel)));
+}
+
/**
* Detect files that have been modified since the last ingest.
* Uses the workspace-local ingest baseline so another workspace cannot advance it.
@@ -91,7 +78,8 @@ function differsFromIngestBaseline(
*/
export function detectStaleFiles(
root: string,
- maxSamples: number = 5
+ maxSamples: number = 5,
+ walkLimit: number = STATUS_WALK_LIMIT,
): StaleInfo {
const workspaceRoot = path.resolve(root);
const baseline = loadIngestBaseline(workspaceRoot);
@@ -104,34 +92,64 @@ export function detectStaleFiles(
staleFiles: 0,
sampleChangedFiles: [],
replayedFiles: [],
+ parseTimeouts: [],
};
}
- const files = collectFiles(workspaceRoot);
- const currentFiles = new Set(files.map((filePath) => path.resolve(filePath)));
+ // Discovery returns canonical paths (macOS `/var` is `/private/var`, a
+ // Windows 8.3 name is expanded), while the baseline's keys sit under the root
+ // as it was given. Compare and display both under the canonical root.
+ const canonicalRoot = canonicalizeDiscoveredFilePath(workspaceRoot);
+ const underCanonicalRoot = (absolutePath: string): string =>
+ canonicalRoot !== workspaceRoot && absolutePath.startsWith(workspaceRoot + path.sep)
+ ? canonicalRoot + absolutePath.slice(workspaceRoot.length)
+ : absolutePath;
+ const ingestedMtimes = new Map<string, number>();
+ for (const [ingestedPath, mtime] of baseline.files) {
+ const absolutePath = path.isAbsolute(ingestedPath)
+ ? path.resolve(ingestedPath)
+ : path.resolve(workspaceRoot, ingestedPath);
+ ingestedMtimes.set(underCanonicalRoot(absolutePath), mtime);
+ }
+
+ const files = collectFiles(workspaceRoot, walkLimit).map(underCanonicalRoot);
+ const currentFiles = new Set(files);
const changedFiles: string[] = [];
+ // Under the canonical root, like everything else compared here.
+ const canonicalSet = (relPaths: readonly string[]): Set<string> =>
+ new Set([...absoluteSet(workspaceRoot, relPaths)].map(underCanonicalRoot));
+ const pending = canonicalSet(baseline.pendingFiles);
+ // Canonical path -> the baseline's own workspace-relative spelling, which is
+ // what status reports: POSIX separators on every platform, as the ingest
+ // summary names them.
+ const timedOut = new Map(
+ baseline.parseTimeouts.map((rel) => [underCanonicalRoot(path.resolve(workspaceRoot, rel)), rel]),
+ );
+ const parseTimeouts: string[] = [];
for (const filePath of files) {
+ const timedOutAs = timedOut.get(filePath);
+ if (timedOutAs !== undefined) {
+ parseTimeouts.push(timedOutAs);
+ continue;
+ }
try {
const stat = fs.statSync(filePath);
if (
- differsFromIngestBaseline(filePath, stat.mtimeMs, baseline.files, baseline.lastIngestAt)
+ pending.has(filePath)
+ || differsFromIngestBaseline(filePath, stat.mtimeMs, ingestedMtimes, baseline.lastIngestAt)
) {
// Make path relative to root for display
- const relative = path.relative(workspaceRoot, filePath);
- changedFiles.push(relative);
+ changedFiles.push(path.relative(canonicalRoot, filePath));
}
} catch {
// skip inaccessible files
}
}
- for (const ingestedPath of baseline.files.keys()) {
- const absolutePath = path.isAbsolute(ingestedPath)
- ? path.resolve(ingestedPath)
- : path.resolve(workspaceRoot, ingestedPath);
+ for (const absolutePath of ingestedMtimes.keys()) {
if (!currentFiles.has(absolutePath) && !fs.existsSync(absolutePath)) {
- changedFiles.push(path.relative(workspaceRoot, absolutePath));
+ changedFiles.push(path.relative(canonicalRoot, absolutePath));
}
}
@@ -143,6 +161,7 @@ export function detectStaleFiles(
staleFiles: changedFiles.length,
sampleChangedFiles: changedFiles.slice(0, maxSamples),
replayedFiles: baseline.replayedFiles,
+ parseTimeouts,
};
}
@@ -196,6 +215,9 @@ export function isFileStale(filePath: string): boolean {
export function createStaleProbe(): (filePath: string) => boolean {
const workspaceRoot = path.resolve(resolveWorkspaceRoot());
const baseline = loadIngestBaseline(workspaceRoot);
+ const pending = baseline
+ ? absoluteSet(workspaceRoot, [...baseline.pendingFiles, ...baseline.parseTimeouts])
+ : new Set<string>();
return (filePath: string): boolean => {
// No baseline means the question this probe answers — "did this file change
@@ -213,6 +235,7 @@ export function createStaleProbe(): (filePath: string) => boolean {
if (!fs.existsSync(absolutePath)) {
return baseline.files.has(absolutePath) || baseline.files.has(filePath);
}
+ if (pending.has(absolutePath)) return true;
try {
return differsFromIngestBaseline(
diff --git a/ix-cli/test/perf/budgets.json b/ix-cli/test/perf/budgets.json
index 6e5dbc52..61a003d7 100644
--- a/ix-cli/test/perf/budgets.json
+++ b/ix-cli/test/perf/budgets.json
@@ -8,7 +8,7 @@
},
"map (no-op)": {
"requests": 6,
- "bytes": 914,
+ "bytes": 1017,
"wallMs": 448
},
"map (one-line TS edit)": {
diff --git a/skills/ix/references/flags.md b/skills/ix/references/flags.md
index 5997c1c1..0b001339 100644
--- a/skills/ix/references/flags.md
+++ b/skills/ix/references/flags.md
@@ -89,6 +89,13 @@ surfaces (#575).
directory inside it refreshes just that part, leaving the rest of the
workspace alone. Only a path in no workspace and no repository becomes a
workspace of its own.
+- **`ix ingest <path>` coalesces with a running `ix map` or `ix ingest` of the
+ same workspace**, as `ix map` does: it does not run beside the holder, asks
+ it for one more pass, and exits 0 (or `IX_MAP_COALESCE_EXIT_CODE`). With
+ `--format json` it prints `{"coalesced": true, "workspace": <root>}`. The
+ holder runs the extra pass before it exits: `ix map` re-ingests the
+ workspace, and so does `ix ingest` once the workspace has a baseline
+ (otherwise it re-ingests only its own path).
- **`ix ingest` honours `--exclude <glob>` and a `.ixignore` at the workspace
root**, matched relative to that root. A deliberate subset of `.gitignore`: `#` comments, `*`, `?`, `**`, a
leading `/` to anchor at the root, a trailing `/` for directories only, and a
diff --git a/skills/ix/references/troubleshooting.md b/skills/ix/references/troubleshooting.md
index 0c0666d7..dba49c2d 100644
--- a/skills/ix/references/troubleshooting.md
+++ b/skills/ix/references/troubleshooting.md
@@ -58,6 +58,15 @@ again.
another writer (a second `ix map` on the same backend) moved the graph
revision under it (default 8). Only a commit that loses every retry is
counted as failed and left for the next run.
+- `IX_PARSE_BUDGET_MS=N` — how long `ix map` / `ix ingest` may spend parsing
+ one file (default 10000). A file that runs past it is skipped and named in
+ the summary (`parseTimeouts` in `--format json`). It is not recorded as
+ ingested: every later run tries it again, and `ix status` warns about it by
+ name (`parseTimeouts` in `--format json`, `parse_timeouts` in `--format llm`)
+ until a run gets it in. `0` turns the budget off.
+- `IX_PARSE_WORKERS=N` — the most parse workers `ix map` / `ix ingest` starts
+ (default 8, never more than the cores less one). Each loads every grammar,
+ so a small edit starts one; raise it for a faster first map on a large machine.
## Harness presence (hermetic reproduction)