Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf(context): bounded git, one batched grep, linear basename maps, incremental BM25 index (#773)

Ian Hock5c4b33f188e3
diff --git a/ix-cli/src/cli/__tests__/bm25-cache.test.ts b/ix-cli/src/cli/__tests__/bm25-cache.test.ts
index 1240dbb6..b952d894 100644
--- a/ix-cli/src/cli/__tests__/bm25-cache.test.ts
+++ b/ix-cli/src/cli/__tests__/bm25-cache.test.ts
@@ -106,6 +106,64 @@ describe("cachedBm25Ranker", () => {
expect(cached[0].path).toBe("src/auth/login.ts");
});
+ it("after a HEAD move, reads only the files whose blob changed", async () => {
+ const repo = gitRepoAccess(dir)!;
+ cachedBm25Ranker(dir)(repo, sourceFiles(repo), QUERIES[0]);
+
+ write("src/auth/session.ts", "export function logout() { validatePassword(''); password }\n");
+ write("src/auth/refresh.ts", "export function refreshSessionToken() { /* races logout */ }\n");
+ git("add", ".");
+ git("commit", "-q", "-m", "touch two files");
+ const moved = gitRepoAccess(dir)!;
+ const files = sourceFiles(moved);
+ const watched = counting(moved);
+
+ for (const q of QUERIES) {
+ expect(cachedBm25Ranker(dir)(q === QUERIES[0] ? watched : moved, files, q)).toEqual(bm25Rank(moved, files, q));
+ }
+ expect(watched.reads.sort()).toEqual(["src/auth/refresh.ts", "src/auth/session.ts"]);
+ });
+
+ it("keys reuse on HEAD's blob, not a staged one the worktree has reverted", async () => {
+ const original = readFileSync(join(dir, "src/list.ts"), "utf-8");
+ const staged = "export function validatePassword() { password password password }\n";
+ // Staged, then the worktree put back to HEAD's text: clean against HEAD,
+ // but the index holds the staged blob.
+ write("src/list.ts", staged);
+ git("add", "src/list.ts");
+ write("src/list.ts", original);
+ const repo = gitRepoAccess(dir)!;
+ const files = sourceFiles(repo);
+ cachedBm25Ranker(dir)(repo, files, QUERIES[0]);
+
+ // Commit what was staged, then bring the worktree up to it.
+ git("commit", "-q", "-m", "commit the staged text");
+ git("checkout", "--", "src/list.ts");
+ const moved = gitRepoAccess(dir)!;
+
+ expect(cachedBm25Ranker(dir)(moved, files, QUERIES[0])).toEqual(bm25Rank(moved, files, QUERIES[0]));
+ });
+
+ it("after a HEAD move that deletes and renames files, scores exactly as bm25Rank", async () => {
+ const repo = gitRepoAccess(dir)!;
+ cachedBm25Ranker(dir)(repo, sourceFiles(repo), QUERIES[0]);
+
+ git("mv", "src/auth/login.ts", "src/auth/signin.ts");
+ git("rm", "-q", "src/list.ts");
+ git("commit", "-q", "-m", "rename and delete");
+ const moved = gitRepoAccess(dir)!;
+ const files = sourceFiles(moved);
+ const watched = counting(moved);
+
+ for (const q of QUERIES) {
+ expect(cachedBm25Ranker(dir)(q === QUERIES[0] ? watched : moved, files, q)).toEqual(bm25Rank(moved, files, q));
+ }
+ expect(watched.reads).toEqual(["src/auth/signin.ts"]);
+ const index = JSON.parse(readFileSync(bm25IndexPath(dir), "utf-8"));
+ expect(index.paths).not.toContain("src/list.ts");
+ expect(index.paths).not.toContain("src/auth/login.ts");
+ });
+
it("rebuilds when HEAD moves", async () => {
const repo = gitRepoAccess(dir)!;
cachedBm25Ranker(dir)(repo, sourceFiles(repo), QUERIES[2]);
@@ -150,7 +208,7 @@ describe("cachedBm25Ranker", () => {
expect(watched.reads).toEqual([]);
});
- it("ignores an index from another version, root or commit, and a corrupt one", async () => {
+ it("ignores an index from another version or root, a malformed one and a corrupt one", async () => {
const repo = gitRepoAccess(dir)!;
const files = sourceFiles(repo);
const rank = cachedBm25Ranker(dir);
@@ -158,7 +216,7 @@ describe("cachedBm25Ranker", () => {
const path = bm25IndexPath(dir);
const good = JSON.parse(readFileSync(path, "utf-8"));
- for (const bad of [{ ...good, v: 0 }, { ...good, root: "/elsewhere" }, { ...good, head: "0".repeat(40) }, "{not json"]) {
+ for (const bad of [{ ...good, v: 0 }, { ...good, root: "/elsewhere" }, { ...good, blobs: [] }, "{not json"]) {
writeFileSync(path, typeof bad === "string" ? bad : JSON.stringify(bad));
const watched = counting(repo);
expect(rank(watched, files, QUERIES[0])).toEqual(bm25Rank(repo, files, QUERIES[0]));
diff --git a/ix-cli/src/cli/__tests__/history.test.ts b/ix-cli/src/cli/__tests__/history.test.ts
index 658187ba..304e8eef 100644
--- a/ix-cli/src/cli/__tests__/history.test.ts
+++ b/ix-cli/src/cli/__tests__/history.test.ts
@@ -13,19 +13,19 @@ import { coChangedFiles, gitRunner, recentCommits, type GitRunner } from "../exp
/** A `git log` that answers from a fixed list of commits, newest first. */
function fakeGit(commits: Array<{ sha: string; date: string; subject: string; files: string[] }>): GitRunner {
- return (args) => {
+ return async (args) => {
const path = args[args.indexOf("--") + 1];
const depth = Number((args.find((a) => /^-\d+$/.test(a)) ?? "-1000").slice(1));
const touching = commits.filter((c) => c.files.includes(path)).slice(0, depth);
if (args.includes("--full-diff")) {
- return touching.map((c) => `@${c.sha}\n\n${c.files.join("\n")}\n`).join("");
+ return touching.map((c) => `\0${c.sha}\n\n${c.files.join("\n")}\n`).join("");
}
return touching.map((c) => `${c.sha}\t${c.date}\t${c.subject}`).join("\n");
};
}
describe("recentCommits", () => {
- it("keeps the newest two, adds the newest fixes, and stays newest first", () => {
+ it("keeps the newest two, adds the newest fixes, and stays newest first", async () => {
// inventory.ts before the rank fix: the fix to the same bug next door was
// behind three feature commits.
const git = fakeGit([
@@ -36,25 +36,25 @@ describe("recentCommits", () => {
{ sha: "97d66ff", date: "2026-05-30", subject: "fix: inventory uses listByKind", files: ["inventory.ts"] },
{ sha: "0000001", date: "2026-05-01", subject: "fix: an older fix", files: ["inventory.ts"] },
]);
- expect(recentCommits(git, "inventory.ts").map((c) => c.sha)).toEqual(["d4ed26f", "7ab9034", "bee214b", "97d66ff"]);
+ expect((await recentCommits(git, "inventory.ts")).map((c) => c.sha)).toEqual(["d4ed26f", "7ab9034", "bee214b", "97d66ff"]);
});
- it("does not repeat a commit that is both newest and a fix", () => {
+ it("does not repeat a commit that is both newest and a fix", async () => {
const git = fakeGit([
{ sha: "a", date: "d", subject: "fix(cli): fail unresolved graph commands (#547)", files: ["resolve.ts"] },
{ sha: "b", date: "d", subject: "perf: cut ix read", files: ["resolve.ts"] },
{ sha: "c", date: "d", subject: "fix(mcp): close leaks", files: ["resolve.ts"] },
]);
- expect(recentCommits(git, "resolve.ts").map((c) => c.sha)).toEqual(["a", "b", "c"]);
+ expect((await recentCommits(git, "resolve.ts")).map((c) => c.sha)).toEqual(["a", "b", "c"]);
});
- it("is empty when git cannot answer", () => {
- expect(recentCommits(() => undefined, "x.ts")).toEqual([]);
+ it("is empty when git cannot answer", async () => {
+ expect(await recentCommits(async () => undefined, "x.ts")).toEqual([]);
});
});
describe("coChangedFiles", () => {
- it("ranks files that changed with the target, discounting sweeps", () => {
+ it("ranks files that changed with the target, discounting sweeps", async () => {
const sweep = Array.from({ length: 25 }, (_, i) => `f${i}.ts`);
const git = fakeGit([
{ sha: "1", date: "d", subject: "s", files: ["inventory.ts", "rank.ts", "api.ts"] },
@@ -64,18 +64,18 @@ describe("coChangedFiles", () => {
{ sha: "5", date: "d", subject: "sweep", files: ["inventory.ts", ...sweep] },
{ sha: "6", date: "d", subject: "sweep", files: ["inventory.ts", ...sweep] },
]);
- const found = coChangedFiles(git, "inventory.ts", new Set());
+ const found = await coChangedFiles(git, "inventory.ts", new Set());
expect(found.map((c) => c.path)).toEqual(["rank.ts", "api.ts"]);
expect(found[0]).toMatchObject({ commits: 2 });
});
- it("skips files the bundle has, and tests unless the target is one", () => {
+ it("skips files the bundle has, and tests unless the target is one", async () => {
const git = fakeGit([
{ sha: "1", date: "d", subject: "s", files: ["a.ts", "b.ts", "src/__tests__/a.test.ts"] },
{ sha: "2", date: "d", subject: "s", files: ["a.ts", "b.ts", "src/__tests__/a.test.ts"] },
]);
- expect(coChangedFiles(git, "a.ts", new Set(["b.ts"]))).toEqual([]);
- expect(coChangedFiles(git, "src/__tests__/a.test.ts", new Set()).map((c) => c.path)).toEqual(["a.ts", "b.ts"]);
+ expect(await coChangedFiles(git, "a.ts", new Set(["b.ts"]))).toEqual([]);
+ expect((await coChangedFiles(git, "src/__tests__/a.test.ts", new Set())).map((c) => c.path)).toEqual(["a.ts", "b.ts"]);
});
});
@@ -86,9 +86,9 @@ describe("gitRunner", () => {
dir = undefined;
});
- it("reads a real repository's log, and returns undefined outside one", () => {
+ it("reads a real repository's log, and returns undefined outside one", async () => {
dir = mkdtempSync(join(tmpdir(), "ix-history-"));
- expect(gitRunner(dir)(["log", "-1"])).toBeUndefined();
+ expect(await gitRunner(dir)(["log", "-1"])).toBeUndefined();
const git = (...args: string[]) => execFileSync("git", args, { cwd: dir, stdio: "ignore" });
git("init", "-q");
git("config", "user.email", "[email protected]");
@@ -100,11 +100,31 @@ describe("gitRunner", () => {
git("commit", "-q", "-m", subject);
}
const run = gitRunner(dir);
- expect(recentCommits(run, "a.ts").map((c) => c.subject)).toEqual(["fix: a handles b", "feat: add a"]);
- expect(coChangedFiles(run, "a.ts", new Set()).map((c) => [c.path, c.commits])).toEqual([["b.ts", 2]]);
+ expect((await recentCommits(run, "a.ts")).map((c) => c.subject)).toEqual(["fix: a handles b", "feat: add a"]);
+ expect((await coChangedFiles(run, "a.ts", new Set())).map((c) => [c.path, c.commits])).toEqual([["b.ts", 2]]);
});
- it("finds co-changes for a workspace in a subdirectory of its repository", () => {
+ it("keeps a commit whole when a path in it contains @", async () => {
+ // The commit marker used to be "@", so `packages/@scope/x.ts` split its
+ // commit and the files after it were credited to nothing.
+ dir = mkdtempSync(join(tmpdir(), "ix-history-"));
+ const git = (...args: string[]) => execFileSync("git", args, { cwd: dir, stdio: "ignore" });
+ git("init", "-q");
+ git("config", "user.email", "[email protected]");
+ git("config", "user.name", "t");
+ mkdirSync(join(dir, "packages", "@scope"), { recursive: true });
+ for (const i of [0, 1]) {
+ writeFileSync(join(dir, "a.ts"), `// ${i}\n`);
+ writeFileSync(join(dir, "packages", "@scope", "x.ts"), `// ${i}\n`);
+ writeFileSync(join(dir, "z.ts"), `// ${i}\n`);
+ git("add", ".");
+ git("commit", "-q", "-m", `change ${i}`);
+ }
+ const found = (await coChangedFiles(gitRunner(dir), "a.ts", new Set(), 5)).map((c) => [c.path, c.commits]);
+ expect(found).toEqual(expect.arrayContaining([["packages/@scope/x.ts", 2], ["z.ts", 2]]));
+ });
+
+ it("finds co-changes for a workspace in a subdirectory of its repository", async () => {
// `git log --name-only` prints paths from the repository's top level, while
// the target path is relative to the workspace root git runs in.
dir = mkdtempSync(join(tmpdir(), "ix-history-"));
@@ -120,12 +140,12 @@ describe("gitRunner", () => {
git("commit", "-q", "-m", `change ${i}`);
}
const run = gitRunner(join(dir, "packages", "web"));
- expect(coChangedFiles(run, "a.ts", new Set()).map((c) => [c.path, c.commits])).toEqual([["b.ts", 2]]);
+ expect((await coChangedFiles(run, "a.ts", new Set())).map((c) => [c.path, c.commits])).toEqual([["b.ts", 2]]);
});
});
describe("recent commits in a bundle", () => {
- it("is one provenance row naming each commit, right after the related files", () => {
+ it("is one provenance row naming each commit, right after the related files", async () => {
const facts: ContextFacts = {
id: "f", name: "resolve.ts", kind: "file", path: "ix-cli/src/cli/resolve.ts",
members: [], memberCount: 0, callerCount: 0, calleeCount: 0, dependentCount: 0, importerCount: 0,
diff --git a/ix-cli/src/cli/__tests__/text-references-batch.test.ts b/ix-cli/src/cli/__tests__/text-references-batch.test.ts
new file mode 100644
index 00000000..0b55edea
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/text-references-batch.test.ts
@@ -0,0 +1,55 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+
+import { collectTextReferences, gitRepoAccess, type RepoAccess } from "../explain/text-references.js";
+
+let dir: string;
+
+function write(path: string, text: string) {
+ mkdirSync(dirname(join(dir, path)), { recursive: true });
+ writeFileSync(join(dir, path), text);
+}
+
+beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), "ix-text-refs-batch-"));
+ write("src/languages.ts", "export const languages = [];\n");
+ write("src/parser.ts", "export function parse() {}\n");
+ write("src/registry.ts", "// loads languages.ts at runtime\nconst table = load('languages.ts');\n");
+ write("scripts/build.ts", "run('src/parser.ts'); run('languages.ts');\n");
+ write("docs/notes.md", "parser.ts and languages.ts are documented here\n");
+ execFileSync("git", ["init", "-q"], { cwd: dir });
+ execFileSync("git", ["add", "."], { cwd: dir });
+});
+
+afterEach(() => {
+ rmSync(dir, { recursive: true, force: true });
+});
+
+describe("collectTextReferences with one batched git grep", () => {
+ it("finds the same references as one grep per name", () => {
+ const batched = gitRepoAccess(dir)!;
+ expect(batched.grepAll).toBeTypeOf("function");
+ let greps = 0;
+ const perName: RepoAccess = {
+ files: () => batched.files(),
+ read: (p) => batched.read(p),
+ grep: (needle) => { greps++; return batched.grep(needle); },
+ };
+ const sources = [
+ { path: "src/languages.ts", role: "target" as const },
+ { path: "src/parser.ts", role: "import" as const },
+ ];
+
+ const one = collectTextReferences(batched, sources, new Set(), 10);
+ const many = collectTextReferences(perName, sources, new Set(), 10);
+
+ expect(one).toEqual(many);
+ expect(one.map((r) => r.path)).toEqual(expect.arrayContaining(["src/registry.ts", "scripts/build.ts"]));
+ expect(greps).toBe(2);
+ });
+});
diff --git a/ix-cli/src/cli/commands/context.ts b/ix-cli/src/cli/commands/context.ts
index 1eaf2258..0caa1e32 100644
--- a/ix-cli/src/cli/commands/context.ts
+++ b/ix-cli/src/cli/commands/context.ts
@@ -1615,8 +1615,8 @@ async function collectHistory(
]) {
if (ref.path) known.add(ref.path);
}
- const recent = recentCommits(git, facts.path);
- const coChanged = await Promise.all(coChangedFiles(git, facts.path, known).map(async (c) => {
+ const [recent, coChanges] = await Promise.all([recentCommits(git, facts.path), coChangedFiles(git, facts.path, known)]);
+ const coChanged = await Promise.all(coChanges.map(async (c) => {
const name = c.path.split("/").pop()!;
const id = await fileNodeId(client, c.path);
const named = `changed with the target in ${c.commits} commits`;
diff --git a/ix-cli/src/cli/explain/bm25-cache.ts b/ix-cli/src/cli/explain/bm25-cache.ts
index 11a0f099..ea6d3862 100644
--- a/ix-cli/src/cli/explain/bm25-cache.ts
+++ b/ix-cli/src/cli/explain/bm25-cache.ts
@@ -37,7 +37,7 @@ import type { RepoAccess } from "./text-references.js";
* same `bm25Score`. Bump INDEX_VERSION whenever `bm25Tokens` or `bm25Doc`
* changes what a document is, or a stale index would score the old way.
*/
-const INDEX_VERSION = 1;
+const INDEX_VERSION = 2;
/**
* What the tokenizer does to a probe that exercises its rules: case, camel
@@ -61,8 +61,11 @@ interface Bm25Index {
/** Clean files that were scored, and each one's length in words. */
paths: string[];
lengths: number[];
+ /** Each scored file's git blob id, so a later HEAD reuses it while unchanged. */
+ blobs: string[];
/** Clean files that were not scored (too large or unreadable at HEAD). */
skipped: string[];
+ skippedBlobs: string[];
/** Every word, to a flat list of [document index, count] pairs. */
postings: Record<string, number[]>;
}
@@ -72,8 +75,13 @@ export interface GitState {
head: string;
/** Tracked paths (relative to the root, as `git ls-files` prints them) that differ from HEAD. */
dirty: ReadonlySet<string>;
+ /** Each path's blob id at HEAD, when known. */
+ blobs?: ReadonlyMap<string, string>;
}
+/** Every git call here is bounded: none may hold up `ix context` for long. */
+const GIT_TIMEOUT_MS = 5_000;
+
/** Path of the index for one workspace root, under IX_HOME. */
export function bm25IndexPath(root: string): string {
const key = createHash("sha256").update(root).digest("hex").slice(0, 12);
@@ -89,13 +97,24 @@ export function readGitState(root: string): GitState | undefined {
try {
const run = (args: string[]) => execFileSync("git", ["--no-optional-locks", ...args], {
cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024,
+ timeout: GIT_TIMEOUT_MS,
});
const head = run(["rev-parse", "--verify", "-q", "HEAD"]).trim();
if (!head) return undefined;
// --relative: paths relative to `root`, like `git ls-files` run there.
// --no-renames: a rename lists both of its paths, not only the new one.
const dirty = run(["diff", "--name-only", "--relative", "--no-renames", "-z", "HEAD", "--"]);
- return { head, dirty: new Set(dirty.split("\0").filter(Boolean)) };
+ // HEAD's blobs, not the index's: a clean file's text is HEAD's, while a
+ // staged change the worktree has since reverted leaves a different blob
+ // in the index, and reusing counts by that blob after it is committed
+ // would score the old text. `<mode> <type> <blob>\t<path>` per entry,
+ // relative to (and limited to) the root, as `git ls-files` prints paths.
+ const blobs = new Map<string, string>();
+ for (const entry of run(["ls-tree", "-r", "-z", head]).split("\0")) {
+ const tab = entry.indexOf("\t");
+ if (tab > 0) blobs.set(entry.slice(tab + 1), entry.slice(0, tab).split(" ")[2]);
+ }
+ return { head, dirty: new Set(dirty.split("\0").filter(Boolean)), blobs };
} catch {
return undefined;
}
@@ -116,10 +135,16 @@ export function cachedBm25Ranker(
const indexPath = opts.indexPath ?? bm25IndexPath(root);
const weights = bm25QueryWeights(query);
const terms = [...weights.keys()];
- const index = loadIndex(indexPath, root, state.head);
- if (index) return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), weights);
+ const index = loadIndex(indexPath, root);
+ if (index && index.head === state.head) {
+ return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), weights);
+ }
- const { docs, built } = buildIndex(repo, files, terms, state, root);
+ // Built for another commit, or not at all. A file whose blob is unchanged
+ // since that index keeps its counts; only the rest are read and tokenised
+ // again. On a HEAD move that touched a handful of files, that is a
+ // handful of reads instead of the whole repository.
+ const { docs, built } = buildIndex(repo, files, terms, state, root, index);
// Only if nothing moved while the files were being read: a file edited in
// that window would otherwise be stored as HEAD's text.
const after = readState();
@@ -162,31 +187,63 @@ function docsFromIndex(
return docs;
}
+/** A previous index's per-file counts, inverted from its postings once. */
+function countsByFile(index: Bm25Index): Array<Map<string, number>> {
+ const counts = index.paths.map(() => new Map<string, number>());
+ for (const term of Object.keys(index.postings)) {
+ const list = index.postings[term];
+ for (let k = 0; k < list.length; k += 2) counts[list[k]]?.set(term, list[k + 1]);
+ }
+ return counts;
+}
+
function buildIndex(
repo: Pick<RepoAccess, "read">,
files: string[],
terms: string[],
state: GitState,
root: string,
+ previous?: Bm25Index,
): { docs: Map<string, Bm25Doc>; built: Bm25Index } {
+ const reusable = previous && state.blobs ? previous : undefined;
+ const prevPosition = new Map((reusable?.paths ?? []).map((path, i) => [path, i]));
+ const prevSkipped = new Map((reusable?.skipped ?? []).map((path, i) => [path, reusable!.skippedBlobs[i]]));
+ let prevCounts: Array<Map<string, number>> | undefined;
const built: Bm25Index = {
v: INDEX_VERSION, tok: TOKENIZER_FINGERPRINT, root, head: state.head, maxBytes: MAX_BM25_BYTES,
- paths: [], lengths: [], skipped: [],
+ paths: [], lengths: [], blobs: [], skipped: [], skippedBlobs: [],
// No prototype: `constructor` and `tostring` are words too.
postings: Object.create(null) as Record<string, number[]>,
};
const docs = new Map<string, Bm25Doc>();
for (const path of files) {
- const full = bm25Doc(path, repo.read(path));
const clean = !state.dirty.has(path);
+ const blob = state.blobs?.get(path) ?? "";
+ let full: Bm25Doc | undefined;
+ if (reusable && clean && blob) {
+ const j = prevPosition.get(path);
+ if (j !== undefined && reusable.blobs[j] === blob) {
+ prevCounts ??= countsByFile(reusable);
+ full = { tf: prevCounts[j], length: reusable.lengths[j] };
+ } else if (prevSkipped.get(path) === blob) {
+ built.skipped.push(path);
+ built.skippedBlobs.push(blob);
+ continue;
+ }
+ }
+ full ??= bm25Doc(path, repo.read(path));
if (!full) {
- if (clean) built.skipped.push(path);
+ if (clean) {
+ built.skipped.push(path);
+ built.skippedBlobs.push(blob);
+ }
continue;
}
if (clean) {
const i = built.paths.length;
built.paths.push(path);
built.lengths.push(full.length);
+ built.blobs.push(blob);
for (const [term, count] of full.tf) (built.postings[term] ??= []).push(i, count);
}
const tf = new Map<string, number>();
@@ -199,12 +256,16 @@ function buildIndex(
return { docs, built };
}
-function loadIndex(path: string, root: string, head: string): Bm25Index | undefined {
+function loadIndex(path: string, root: string): Bm25Index | undefined {
try {
const data = JSON.parse(readFileSync(path, "utf-8")) as Partial<Bm25Index>;
- if (data.v !== INDEX_VERSION || data.tok !== TOKENIZER_FINGERPRINT || data.root !== root || data.head !== head || data.maxBytes !== MAX_BM25_BYTES) return undefined;
+ if (data.v !== INDEX_VERSION || data.tok !== TOKENIZER_FINGERPRINT || data.root !== root || data.maxBytes !== MAX_BM25_BYTES) return undefined;
+ if (typeof data.head !== "string") return undefined;
if (!Array.isArray(data.paths) || !Array.isArray(data.lengths) || !Array.isArray(data.skipped)) return undefined;
- if (data.paths.length !== data.lengths.length || !data.postings || typeof data.postings !== "object") return undefined;
+ if (!Array.isArray(data.blobs) || !Array.isArray(data.skippedBlobs)) return undefined;
+ if (data.paths.length !== data.lengths.length || data.paths.length !== data.blobs.length) return undefined;
+ if (data.skipped.length !== data.skippedBlobs.length) return undefined;
+ if (!data.postings || typeof data.postings !== "object") return undefined;
return data as Bm25Index;
} catch {
return undefined;
diff --git a/ix-cli/src/cli/explain/history.ts b/ix-cli/src/cli/explain/history.ts
index 0778be57..01503f37 100644
--- a/ix-cli/src/cli/explain/history.ts
+++ b/ix-cli/src/cli/explain/history.ts
@@ -1,6 +1,6 @@
// Copyright 2026 Ix Infrastructure Inc.
-import { execFileSync } from "node:child_process";
+import { execFile } from "node:child_process";
import { isTestPath } from "./related-files.js";
@@ -40,8 +40,15 @@ export interface CoChange {
score: number;
}
-/** Runs `git <args>` in the repository and returns stdout, or undefined. */
-export type GitRunner = (args: string[]) => string | undefined;
+/** Runs `git <args>` in the repository and resolves stdout, or undefined. */
+export type GitRunner = (args: string[]) => Promise<string | undefined>;
+
+/**
+ * How long one `git log` may take. Asynchronous and bounded: on a large
+ * history the synchronous call blocked `ix context` -- and the MCP timer that
+ * is meant to cut it off -- for as long as git cared to take.
+ */
+const GIT_TIMEOUT_MS = 5_000;
/** Newest commits shown, whatever they are. */
export const RECENT_COMMITS = 2;
@@ -66,8 +73,8 @@ const MIN_CO_CHANGES = 2;
* same bug in `inventory.ts` (#229) was that file's fifth commit, behind
* three features. A fix is where a file's bug knowledge lives.
*/
-export function recentCommits(git: GitRunner, path: string): CommitRef[] {
- const out = git(["log", `-${RECENT_WINDOW}`, "--format=%h%x09%ad%x09%s", "--date=short", "--", path]);
+export async function recentCommits(git: GitRunner, path: string): Promise<CommitRef[]> {
+ const out = await git(["log", `-${RECENT_WINDOW}`, "--format=%h%x09%ad%x09%s", "--date=short", "--", path]);
if (!out) return [];
const all = out.split("\n").filter(Boolean).map((line) => {
const [sha, date, ...subject] = line.split("\t");
@@ -80,23 +87,27 @@ export function recentCommits(git: GitRunner, path: string): CommitRef[] {
return all.filter((c) => chosen.has(c));
}
-export function coChangedFiles(
+export async function coChangedFiles(
git: GitRunner,
path: string,
exclude: ReadonlySet<string>,
limit = MAX_CO_CHANGES,
-): CoChange[] {
+): Promise<CoChange[]> {
// --full-diff lists every file each commit touched, not only `path`: one
// call instead of one `git show` per commit. --relative prints them relative
// to the workspace root git runs in, as `path` is: without it they were
// relative to the repository's top level, so a workspace in a subdirectory
// of its repo (`ix map packages/web`) never matched `path` and had no
// co-changes at all.
- const out = git(["log", `-${HISTORY_DEPTH}`, "--format=@%h", "--name-only", "--relative", "--full-diff", "--", path]);
+ //
+ // Each commit starts with a NUL before its hash. It used to be "@", which a
+ // path can contain (`packages/@scope/x.ts`): such a path split its commit in
+ // two and its files were credited to nothing.
+ const out = await git(["log", `-${HISTORY_DEPTH}`, "--format=%x00%h", "--name-only", "--relative", "--full-diff", "--", path]);
if (!out) return [];
const targetIsTest = isTestPath(path);
const tally = new Map<string, CoChange>();
- for (const block of out.split("@").slice(1)) {
+ for (const block of out.split("\0").slice(1)) {
const files = [...new Set(block.split("\n").slice(1).map((l) => l.trim()).filter(Boolean))];
if (files.length > MAX_COMMIT_FILES || !files.includes(path)) continue;
for (const file of files) {
@@ -115,13 +126,13 @@ export function coChangedFiles(
/** `git` in `root`, or undefined when it fails (not a checkout, no history). */
export function gitRunner(root: string): GitRunner {
- return (args) => {
- try {
- return execFileSync("git", args, {
- cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 16 * 1024 * 1024,
- });
- } catch {
- return undefined;
- }
- };
+ return (args) =>
+ new Promise((resolve) => {
+ execFile(
+ "git",
+ args,
+ { cwd: root, encoding: "utf-8", maxBuffer: 16 * 1024 * 1024, timeout: GIT_TIMEOUT_MS },
+ (err, stdout) => resolve(err ? undefined : stdout),
+ );
+ });
}
diff --git a/ix-cli/src/cli/explain/issue.ts b/ix-cli/src/cli/explain/issue.ts
index 33b25630..bfb3e327 100644
--- a/ix-cli/src/cli/explain/issue.ts
+++ b/ix-cli/src/cli/explain/issue.ts
@@ -201,8 +201,12 @@ export async function pickStartingPoints(
const tracked = new Set(deps.files);
const byBasename = new Map<string, string[]>();
for (const file of deps.files) {
+ // push, not a spread: copying the list on every insert was quadratic in
+ // the number of files sharing a basename (index.ts, README.md).
const base = posix.basename(file);
- byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
+ const list = byBasename.get(base);
+ if (list) list.push(file);
+ else byBasename.set(base, [file]);
}
for (const token of paths) {
if (starts.length >= max) return { starts, unresolved };
diff --git a/ix-cli/src/cli/explain/text-references.ts b/ix-cli/src/cli/explain/text-references.ts
index e17e15a9..d040fb3f 100644
--- a/ix-cli/src/cli/explain/text-references.ts
+++ b/ix-cli/src/cli/explain/text-references.ts
@@ -56,6 +56,11 @@ export interface RepoAccess {
read(path: string): string | undefined;
/** Files containing `needle` literally, with the matching lines. */
grep(needle: string): Array<{ path: string; line: string }>;
+ /**
+ * `grep` for several needles in one pass, keyed by needle. Optional: when
+ * present, a bundle's reverse lookups cost one search instead of one each.
+ */
+ grepAll?(needles: string[]): Map<string, Array<{ path: string; line: string }>>;
}
/** Text references a bundle carries. */
@@ -92,8 +97,12 @@ export function collectTextReferences(
const tracked = new Set(files);
const byBasename = new Map<string, string[]>();
for (const file of files) {
+ // push, not a spread: copying the list on every insert was quadratic in
+ // the number of files sharing a basename (index.ts, README.md).
const base = posix.basename(file);
- byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
+ const list = byBasename.get(base);
+ if (list) list.push(file);
+ else byBasename.set(base, [file]);
}
const sourcePaths = new Set(sources.map((s) => s.path));
const found = new Map<string, TextReference>();
@@ -103,6 +112,14 @@ export function collectTextReferences(
if (!prior || score > prior.score) found.set(path, { path, score, reason });
};
+ // Every reverse lookup the loop below will make, in one search when the
+ // repository can do that.
+ const reverseNames = [...new Set(sources
+ .map((s) => posix.basename(s.path))
+ .filter((name) => (byBasename.get(name) ?? []).length === 1))];
+ const grepped = repo.grepAll && reverseNames.length > 0 ? repo.grepAll(reverseNames) : undefined;
+ const grep = (name: string) => grepped?.get(name) ?? (grepped ? [] : repo.grep(name));
+
for (const source of sources) {
const text = repo.read(source.path);
if (text === undefined) continue;
@@ -124,7 +141,7 @@ export function collectTextReferences(
// shares can be attributed from a bare mention.
if ((byBasename.get(name) ?? []).length !== 1) continue;
const mentioners = new Set<string>();
- for (const hit of repo.grep(name)) {
+ for (const hit of grep(name)) {
if (hit.path === source.path || !tracked.has(hit.path) || !CODE_FILE.test(hit.path)) continue;
if (IMPORT_LINE.test(hit.line)) continue;
mentioners.add(hit.path);
@@ -169,6 +186,36 @@ export function resolveToken(
return candidates.size === 1 ? [...candidates][0] : undefined;
}
+/** How long one git call on the bundle path may take. */
+const GIT_TIMEOUT_MS = 10_000;
+
+/**
+ * `git grep -F` for every needle at once, with each matching line credited to
+ * every needle it contains: the same answer as one literal search per needle,
+ * for one process instead of up to nine.
+ */
+function gitGrep(root: string, needles: string[]): Map<string, Array<{ path: string; line: string }>> {
+ const hits = new Map<string, Array<{ path: string; line: string }>>(needles.map((n) => [n, []]));
+ if (needles.length === 0) return hits;
+ let out: string;
+ try {
+ out = execFileSync("git", ["grep", "-n", "-I", "-F", ...needles.flatMap((n) => ["-e", n])], {
+ cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 16 * 1024 * 1024,
+ timeout: GIT_TIMEOUT_MS,
+ });
+ } catch {
+ return hits; // exit 1 is "no match"; a timeout is no answer
+ }
+ for (const row of out.split("\n")) {
+ if (!row) continue;
+ const first = row.indexOf(":");
+ const second = row.indexOf(":", first + 1);
+ const hit = { path: row.slice(0, first), line: row.slice(second + 1) };
+ for (const needle of needles) if (hit.line.includes(needle)) hits.get(needle)!.push(hit);
+ }
+ return hits;
+}
+
/** A path and the source paths its build output could have come from. */
const BUILD_EXTENSIONS: Array<[RegExp, string[]]> = [
[/\.js$/, [".ts", ".tsx", ".js"]],
@@ -200,6 +247,7 @@ export function gitRepoAccess(root: string): RepoAccess | undefined {
try {
listing = execFileSync("git", ["ls-files", "-z"], {
cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024,
+ timeout: GIT_TIMEOUT_MS,
}).split("\0").filter(Boolean);
} catch {
return undefined;
@@ -221,19 +269,7 @@ export function gitRepoAccess(root: string): RepoAccess | undefined {
if (fd !== undefined) closeSync(fd);
}
},
- grep: (needle) => {
- try {
- const out = execFileSync("git", ["grep", "-n", "-I", "-F", "-e", needle], {
- cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 16 * 1024 * 1024,
- });
- return out.split("\n").filter(Boolean).map((row) => {
- const first = row.indexOf(":");
- const second = row.indexOf(":", first + 1);
- return { path: row.slice(0, first), line: row.slice(second + 1) };
- });
- } catch {
- return []; // exit 1 is "no match"
- }
- },
+ grep: (needle) => gitGrep(root, [needle]).get(needle) ?? [],
+ grepAll: (needles) => gitGrep(root, needles),
};
}