Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

Merge origin/main into perf/query-paths

KageBinary6d708bedee79
diff --cc ix-cli/src/cli/commands/context.ts
index 7200f908,ded9180c..aeeaf67f
--- a/ix-cli/src/cli/commands/context.ts
+++ b/ix-cli/src/cli/commands/context.ts
@@@ -1632,32 -1671,10 +1673,10 @@@ async function buildIssueBundle
opts: ContextOptions,
budgets: BudgetSnapshot,
): Promise<ContextBundle | undefined> {
- let text: string;
- try {
- text = await readIssueText(arg);
- } catch (error) {
- issueFailure(
- "issue_unreadable",
- `Cannot read the issue from ${arg === "-" ? "stdin" : `"${arg}"`}: ${(error as Error).message}`,
- opts.format,
- );
- return undefined;
- }
- if (!text.trim()) {
- issueFailure("empty_issue", `The issue ${arg === "-" ? "on stdin" : `in "${arg}"`} is empty.`, opts.format);
- return undefined;
- }
-
+ const text = await readIssueOrReport(arg, opts.format);
+ if (text === undefined) return undefined;
- const client = new IxClient(getEndpoint());
+ const client = new IxClient(getEndpoint(), undefined, QUERY_CLIENT_OPTIONS);
- await ensureReadScope(client);
- const scope = activeReadScope();
- const root = resolveWorkspaceRoot();
- const plan = await planIssue(text, {
- repo: gitRepoAccess(root),
- rank: cachedBm25Ranker(root),
- search: async (name) =>
- (await client.search(name, { limit: ISSUE_SEARCH_LIMIT, nameOnly: true, ...scope })).map(symbolHit),
- });
+ const plan = await planIssueWith(client, text);
// A path start has no node yet, and the graph may not have one either: it
// does not index every tracked file. chooseCentre walks BM25 for one it does.
const { starts, centre, walked } = await chooseCentre(
@@@ -1711,6 -1730,95 +1732,99 @@@
});
}
+ /** The issue's text, or undefined after reporting why it has none. */
+ async function readIssueOrReport(arg: string, format: string | undefined): Promise<string | undefined> {
+ let text: string;
+ try {
+ text = await readIssueText(arg);
+ } catch (error) {
+ issueFailure(
+ "issue_unreadable",
+ `Cannot read the issue from ${arg === "-" ? "stdin" : `"${arg}"`}: ${(error as Error).message}`,
+ format,
+ );
+ return undefined;
+ }
+ if (!text.trim()) {
+ issueFailure("empty_issue", `The issue ${arg === "-" ? "on stdin" : `in "${arg}"`} is empty.`, format);
+ return undefined;
+ }
+ return text;
+ }
+
+ /** The issue's starting points and lexical ranking, against this workspace's graph. */
+ async function planIssueWith(client: IxClient, text: string) {
+ await ensureReadScope(client);
+ const scope = activeReadScope();
++ const root = resolveWorkspaceRoot();
+ return planIssue(text, {
- repo: gitRepoAccess(resolveWorkspaceRoot()),
++ repo: gitRepoAccess(root),
++ // Through the on-disk term index (bm25-cache.ts): the same ranking, with
++ // only the files that differ from HEAD read again.
++ rank: cachedBm25Ranker(root),
+ search: async (name) =>
+ (await client.search(name, { limit: ISSUE_SEARCH_LIMIT, nameOnly: true, ...scope })).map(symbolHit),
+ });
+ }
+
+ /**
+ * `ix context --from-issue --lean`: the starting points Ix trusts and the
+ * next few files, with no bundle around them. See `leanIssueView` for why.
+ * Nothing here needs a centre node, so it is also the cheap path: one search
+ * per name the issue mentions, and BM25 over the tracked files.
+ */
+ async function emitLeanIssue(arg: string, opts: ContextOptions): Promise<void> {
+ const text = await readIssueOrReport(arg, opts.format);
+ if (text === undefined) return;
- const view = leanIssueView(await planIssueWith(new IxClient(getEndpoint()), text));
++ const view = leanIssueView(await planIssueWith(new IxClient(getEndpoint(), undefined, QUERY_CLIENT_OPTIONS), text));
+ if (opts.format === "json") {
+ printJson({ kind: "issue_lean", ...view });
+ } else if (opts.format === "llm") {
+ printLlmLines(renderLeanIssueLlm(view));
+ } else {
+ console.log(renderLeanIssueText(view));
+ }
+ }
+
+ function spanOf(s: StartingPoint): string {
+ return s.lineStart !== undefined ? `${s.path}:${s.lineStart}-${s.lineEnd ?? s.lineStart}` : s.path;
+ }
+
+ /** Why a start was trusted, in the words an agent reads. */
+ function startWhy(s: StartingPoint): string {
+ return s.via === "path in issue" ? "the issue names this file" : `the issue names \`${s.token}\``;
+ }
+
+ export function renderLeanIssueText(view: LeanIssueView): string {
+ if (view.confidence === "low") {
+ return `Ix found no confident starting point for this issue: ${view.reason}.`;
+ }
+ const lines = ["Where to start (Ix resolved these from the issue's own words):"];
+ for (const s of view.startingPoints) {
+ lines.push(` ${spanOf(s)} ${s.kind === "file" ? "" : `${s.name} (${s.kind}) — `}${startWhy(s)}`);
+ }
+ if (view.alsoRanked.length > 0) {
+ lines.push(`Then, by the issue's text: ${view.alsoRanked.map((f) => f.path).join(", ")}`);
+ }
+ return lines.join("\n");
+ }
+
+ export function renderLeanIssueLlm(view: LeanIssueView): string[] {
+ const out = [llmLine("issue", [
+ ["confidence", view.confidence],
+ ...(view.reason ? [["reason", view.reason] as [string, string]] : []),
+ ])];
+ for (const s of view.startingPoints) {
+ out.push(llmLine("start", [
+ ["path", s.path],
+ ...(s.lineStart !== undefined ? [["lines", `${s.lineStart}-${s.lineEnd ?? s.lineStart}`] as [string, string]] : []),
+ ["name", s.name], ["kind", s.kind], ["why", startWhy(s)],
+ ]));
+ }
+ for (const f of view.alsoRanked) out.push(llmLine("ranked", [["path", f.path]]));
+ return out;
+ }
+
/**
* A `--from-issue` failure, readable by the format asked for. `reportFailure`
* writes nothing to stdout under `--format json`, so a JSON caller got an empty
diff --cc ix-cli/src/cli/commands/explain.ts
index c254db3a,4fc9730f..589ef666
--- a/ix-cli/src/cli/commands/explain.ts
+++ b/ix-cli/src/cli/commands/explain.ts
@@@ -2,10 -2,9 +2,10 @@@
import type { Command } from "commander";
import { IxClient } from "../../client/api.js";
+import { QUERY_CLIENT_OPTIONS } from "../../client/request-memo.js";
import { getEndpoint } from "../config.js";
import { formatExplain, relativePath, printJson, type ExplainResult, type EntityRef, type Diagnostic } from "../format.js";
- import { resolveFileOrReport, isRawId } from "../resolve.js";
+ import { resolveFileOrReport, isRawId, activeReadScope } from "../resolve.js";
import { isFileStale } from "../stale.js";
import { collectFacts } from "../explain/facts.js";
import { inferRole } from "../explain/role-inference.js";
diff --cc ix-cli/src/cli/commands/impact.ts
index a6ad1c8a,5af5eade..756568b3
--- a/ix-cli/src/cli/commands/impact.ts
+++ b/ix-cli/src/cli/commands/impact.ts
@@@ -2,14 -2,14 +2,15 @@@
import type { Command } from "commander";
import chalk from "chalk";
- import { renderSection, renderKeyValue, renderNote, renderResolvedHeader, colorizeKind } from "../ui.js";
+ import { renderSection, renderKeyValue, renderNote, renderResolvedHeader, renderWarning, colorizeKind } from "../ui.js";
import { IxClient } from "../../client/api.js";
+import { QUERY_CLIENT_OPTIONS } from "../../client/request-memo.js";
import { getEndpoint } from "../config.js";
- import { resolveFileOrReport, printResolved } from "../resolve.js";
+ import { activeReadScope, resolveFileOrReport, printResolved } from "../resolve.js";
+ import { checkGraphHealth, graphHealthJson, graphHealthLlmFields, graphHealthProse, isUnhealthy } from "../graph-health.js";
import { bucketByHierarchy, getSystemPath, formatSystemPath, hasMapData, type SystemPath } from "../hierarchy.js";
import { inferRiskSemantics, humanizeLabel, type ImpactFacts, type RiskSemantics } from "../impact/risk-semantics.js";
- import { lineSpan, printJson, rowLocation, stripNulls } from "../format.js";
+ import { lineSpan, printJson, relativePath, rowLocation, stripNulls } from "../format.js";
import { llmLine } from "../llm.js";
import { parsePickOption } from "../options.js";
diff --cc ix-cli/src/cli/context-bundle-schema.ts
index 85e74e29,6f3be6f1..ba7d85d0
--- a/ix-cli/src/cli/context-bundle-schema.ts
+++ b/ix-cli/src/cli/context-bundle-schema.ts
@@@ -51,170 -74,125 +51,182 @@@ function buildSchemas(z: typeof Zod.z)
id: z.string(),
name: z.string(),
kind: z.string(),
+ resolutionMode: z.string(),
path: z.string().optional(),
- lineStart: z.number().int().positive().optional(),
- lineEnd: z.number().int().positive().optional(),
- stale: z.boolean(),
}),
- ),
- relationships: z.array(z.object({ src: z.string(), dst: z.string(), predicate: z.string() })),
- claims: z.array(
- z
- .object({ id: z.string(), entityId: z.string(), statement: z.string(), status: z.string() })
+ // Both optional, and only on a bundle built with `--from-issue`: every
+ // bundle saved before them, and every bundle built from a named target,
+ // still validates. Declared rather than left out, because zod strips an
+ // undeclared key on the way to disk and on the way back.
+ issue: z
+ .object({
+ startingPoints: z.array(
+ z.object({
+ token: z.string(),
+ id: z.string().optional(),
+ name: z.string(),
+ kind: z.string(),
+ path: z.string(),
+ lineStart: z.number().int().positive().optional(),
+ lineEnd: z.number().int().positive().optional(),
+ via: z.enum(["path in issue", "identifier in issue", "bm25 fallback"]),
+ }),
+ ),
+ unresolved: z.array(z.string()),
+ fallback: z.boolean(),
+ })
+ .optional(),
+ rankedFiles: z
+ .array(z.object({ path: z.string(), score: z.number(), reason: z.string() }))
+ .optional(),
+ entities: z.array(
+ z.object({
+ id: z.string(),
+ name: z.string(),
+ kind: z.string(),
+ path: z.string().optional(),
+ lineStart: z.number().int().positive().optional(),
+ lineEnd: z.number().int().positive().optional(),
+ stale: z.boolean(),
+ }),
+ ),
+ relationships: z.array(z.object({ src: z.string(), dst: z.string(), predicate: z.string() })),
+ claims: z.array(
+ z
+ .object({ id: z.string(), entityId: z.string(), statement: z.string(), status: z.string() })
+ .catchall(z.unknown()),
+ ),
+ // DecisionReport/ConflictReport/IntentReport are the backend's own contracts
+ // with their own shapes; they are forwarded, never dereferenced field-by-field
+ // here, so they stay loose rather than duplicating those types in a second
+ // place that could drift from them.
+ decisions: z.array(z.record(z.string(), z.unknown())),
+ conflicts: z.array(z.record(z.string(), z.unknown())),
+ intents: z.array(z.record(z.string(), z.unknown())),
+ provenance: z
+ .object({
+ sourceUri: z.string().optional(),
+ sourceHash: z.string().optional(),
+ extractor: z.string().optional(),
+ sourceType: z.string().optional(),
+ observedAt: z.string().optional(),
+ introducedRev: z.number().optional(),
+ historyLength: z.number(),
+ stale: z.boolean(),
+ })
.catchall(z.unknown()),
- ),
- // DecisionReport/ConflictReport/IntentReport are the backend's own contracts
- // with their own shapes; they are forwarded, never dereferenced field-by-field
- // here, so they stay loose rather than duplicating those types in a second
- // place that could drift from them.
- decisions: z.array(z.record(z.string(), z.unknown())),
- conflicts: z.array(z.record(z.string(), z.unknown())),
- intents: z.array(z.record(z.string(), z.unknown())),
- provenance: z
- .object({
- sourceUri: z.string().optional(),
- sourceHash: z.string().optional(),
- extractor: z.string().optional(),
- sourceType: z.string().optional(),
- observedAt: z.string().optional(),
- introducedRev: z.number().optional(),
- historyLength: z.number(),
- stale: z.boolean(),
- })
- .catchall(z.unknown()),
- freshness: z.object({ stale: z.boolean(), classification: z.string() }),
- // Only when the graph the bundle was drawn from is degraded or empty (see
- // graph-health.ts). Optional and additive, like `issue`; declared so zod
- // does not strip it on the way to disk.
- graph: z
- .object({
- status: z.string(),
- reason: z.string().optional(),
- message: z.string().optional(),
- fix: z.string().optional(),
- })
- .catchall(z.unknown())
- .optional(),
- evidence: z.array(
- z.object({
- id: z.string(),
- kind: z.string(),
- source: z.string(),
- title: z.string(),
- score: z.number(),
- reason: z.string(),
- refs: z.array(z.string()),
- // Optional and additive: bundles saved before it existed still parse.
- // Listed rather than left to `catchall`, because an unlisted key is
- // stripped on the way to disk and on the way back.
- location: z
- .object({
- path: z.string(),
- lineStart: z.number().int().positive().optional(),
- lineEnd: z.number().int().positive().optional(),
- })
- .optional(),
+ freshness: z.object({ stale: z.boolean(), classification: z.string() }),
++ // Only when the graph the bundle was drawn from is degraded or empty (see
++ // graph-health.ts). Optional and additive, like `issue`; declared so zod
++ // does not strip it on the way to disk.
++ graph: z
++ .object({
++ status: z.string(),
++ reason: z.string().optional(),
++ message: z.string().optional(),
++ fix: z.string().optional(),
++ })
++ .catchall(z.unknown())
++ .optional(),
+ evidence: z.array(
+ z.object({
+ id: z.string(),
+ kind: z.string(),
+ source: z.string(),
+ title: z.string(),
+ score: z.number(),
+ reason: z.string(),
+ refs: z.array(z.string()),
+ // Optional and additive: bundles saved before it existed still parse.
+ // Listed rather than left to `catchall`, because an unlisted key is
+ // stripped on the way to disk and on the way back.
+ location: z
+ .object({
+ path: z.string(),
+ lineStart: z.number().int().positive().optional(),
+ lineEnd: z.number().int().positive().optional(),
+ })
+ .optional(),
+ }),
+ ),
+ budgets: z.object({
+ maxEntities: z.number().int().positive(),
+ maxRelationships: z.number().int().positive(),
+ maxEvidence: z.number().int().positive(),
+ // Optional, and only this one. A bundle built before `--max-tokens`
+ // existed carries the other four and nothing else, and zod strips keys it
+ // does not declare — so leaving it out here would not have been "tolerant
+ // of old files", it would have silently dropped the field from every new
+ // one on its way to disk. Required would refuse to load those old files
+ // instead, which is a worse answer than reporting `tokens=not-given`.
+ maxTokens: z.number().int().positive().optional(),
+ maxChars: z.number().int().positive(),
}),
- ),
- budgets: z.object({
- maxEntities: z.number().int().positive(),
- maxRelationships: z.number().int().positive(),
- maxEvidence: z.number().int().positive(),
- // Optional, and only this one. A bundle built before `--max-tokens`
- // existed carries the other four and nothing else, and zod strips keys it
- // does not declare — so leaving it out here would not have been "tolerant
- // of old files", it would have silently dropped the field from every new
- // one on its way to disk. Required would refuse to load those old files
- // instead, which is a worse answer than reporting `tokens=not-given`.
- maxTokens: z.number().int().positive().optional(),
- maxChars: z.number().int().positive(),
- }),
- truncation: z.object({
- // Optional for the same reason `budgets.maxTokens` is: a bundle saved
- // before this field existed carries the counters and nothing else, and an
- // undeclared key would be stripped from every new one on its way to disk.
- cut: z.array(z.object({ what: z.string(), count: z.number().int().positive() })).optional(),
- entitiesTruncated: z.number().int().nonnegative(),
- relationshipsTruncated: z.number().int().nonnegative(),
- evidenceTruncated: z.number().int().nonnegative(),
- charactersTruncated: z.number().int().nonnegative(),
- // Optional on the way in, always written on the way out: a bundle saved
- // before the conflict cap existed has no such counter, and refusing to
- // read it back would break `--diff` against every investigation on disk.
- conflictsTruncated: z.number().int().nonnegative().optional(),
- }),
- // `asOfRev` and `depth` are the two saved values `ix context --diff` re-sends
- // to the backend, so they are typed rather than accepted as anything.
- // Unknown keys pass through: metadata is the bundle's extension point, and
- // stripping them would silently drop data on the read-back path.
- metadata: z
- .object({
- asOfRev: z.number().optional(),
- depth: z.string().optional(),
- rankingRule: z.string(),
- })
- .catchall(z.unknown()),
-});
+ truncation: z.object({
+ // Optional for the same reason `budgets.maxTokens` is: a bundle saved
+ // before this field existed carries the counters and nothing else, and an
+ // undeclared key would be stripped from every new one on its way to disk.
+ cut: z.array(z.object({ what: z.string(), count: z.number().int().positive() })).optional(),
+ entitiesTruncated: z.number().int().nonnegative(),
+ relationshipsTruncated: z.number().int().nonnegative(),
+ evidenceTruncated: z.number().int().nonnegative(),
+ charactersTruncated: z.number().int().nonnegative(),
+ // Optional on the way in, always written on the way out: a bundle saved
+ // before the conflict cap existed has no such counter, and refusing to
+ // read it back would break `--diff` against every investigation on disk.
+ conflictsTruncated: z.number().int().nonnegative().optional(),
+ }),
+ // `asOfRev` and `depth` are the two saved values `ix context --diff` re-sends
+ // to the backend, so they are typed rather than accepted as anything.
+ // Unknown keys pass through: metadata is the bundle's extension point, and
+ // stripping them would silently drop data on the read-back path.
+ metadata: z
+ .object({
+ asOfRev: z.number().optional(),
+ depth: z.string().optional(),
+ rankingRule: z.string(),
+ })
+ .catchall(z.unknown()),
+ });
-/**
- * Contract for the on-disk saved-investigation envelope (`~/.ix/investigations`).
- *
- * `id` and `savedAt` are rendered to the terminal and copied into the emitted
- * `ix-investigation-diff/1` JSON, so they are validated here alongside the
- * bundle rather than trusted from a bare cast.
- */
-export const savedInvestigationSchema = z.object({
- schema: z.literal(INVESTIGATION_SCHEMA),
- // `sanitizeId` percent-escapes anything outside this set on the way in, so a
- // stored id can only contain these characters. Pinning the same set on the
- // way out keeps control characters and ANSI escapes out of an id that
- // `renderNote`/`renderSection` write straight to the terminal.
- id: z.string().regex(/^[A-Za-z0-9._~-]+$/),
- savedAt: z.string(),
- bundle: contextBundleSchema,
-});
+ /**
+ * Contract for the on-disk saved-investigation envelope (`~/.ix/investigations`).
+ *
+ * `id` and `savedAt` are rendered to the terminal and copied into the emitted
+ * `ix-investigation-diff/1` JSON, so they are validated here alongside the
+ * bundle rather than trusted from a bare cast.
+ */
+ const savedInvestigationSchema = z.object({
+ schema: z.literal(INVESTIGATION_SCHEMA),
+ // `sanitizeId` percent-escapes anything outside this set on the way in, so a
+ // stored id can only contain these characters. Pinning the same set on the
+ // way out keeps control characters and ANSI escapes out of an id that
+ // `renderNote`/`renderSection` write straight to the terminal.
+ id: z.string().regex(/^[A-Za-z0-9._~-]+$/),
+ savedAt: z.string(),
+ bundle: contextBundleSchema,
+ });
+
+ return { contextBundleSchema, savedInvestigationSchema };
+}
+
+let built: ReturnType<typeof buildSchemas> | undefined;
+function schemas(): ReturnType<typeof buildSchemas> {
+ return (built ??= buildSchemas((requireZod("zod") as typeof Zod).z));
+}
+
+type BundleSchema = ReturnType<typeof buildSchemas>["contextBundleSchema"];
+type InvestigationSchema = ReturnType<typeof buildSchemas>["savedInvestigationSchema"];
+
+/** The bundle contract (see `buildSchemas`); zod is loaded on the first parse. */
+export const contextBundleSchema = {
+ parse: (value: unknown): Zod.infer<BundleSchema> => schemas().contextBundleSchema.parse(value),
+ safeParse: (value: unknown): ReturnType<BundleSchema["safeParse"]> => schemas().contextBundleSchema.safeParse(value),
+};
+
+/** The saved-investigation contract (see `buildSchemas`); zod is loaded on the first parse. */
+export const savedInvestigationSchema = {
+ parse: (value: unknown): Zod.infer<InvestigationSchema> => schemas().savedInvestigationSchema.parse(value),
+ safeParse: (value: unknown): ReturnType<InvestigationSchema["safeParse"]> =>
+ schemas().savedInvestigationSchema.safeParse(value),
+};
diff --cc ix-cli/src/cli/explain/bm25-cache.ts
index 1ed30486,00000000..11a0f099
mode 100644,000000..100644
--- a/ix-cli/src/cli/explain/bm25-cache.ts
+++ b/ix-cli/src/cli/explain/bm25-cache.ts
@@@ -1,230 -1,0 +1,231 @@@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { ixHome } from "../ix-home.js";
+import {
+ bm25Doc,
- bm25QueryTerms,
++ bm25QueryWeights,
+ bm25Rank,
+ bm25Score,
+ bm25Tokens,
+ MAX_BM25_BYTES,
+ type Bm25Doc,
+ type Bm25Hit,
+} from "./issue.js";
+import type { RepoAccess } from "./text-references.js";
+
+/**
+ * A term index of the tracked source files, so `ix context --from-issue` does
+ * not read and tokenize every one of them on every call.
+ *
+ * Without it BM25 opens every tracked source file (up to 200 KB each) per
+ * issue: measured at 56 ms on the Ix repository and ~400 ms on apache/dubbo
+ * (2,427 files), paid again for the next issue against the same checkout.
+ *
+ * Keyed by the commit, so the index describes the files AS COMMITTED: it is
+ * built from the working tree, but only files that match HEAD are stored, and
+ * files that differ from HEAD (`git diff HEAD`, staged or not) are always read
+ * fresh. Git decides what is dirty, including the racily-clean case a size and
+ * mtime check would miss, so a hit scores exactly the text a fresh read would.
+ * One index per workspace root, replaced when HEAD moves.
+ *
+ * Scores are bit-identical to `bm25Rank`: the same documents go through the
+ * same `bm25Score`. Bump INDEX_VERSION whenever `bm25Tokens` or `bm25Doc`
+ * changes what a document is, or a stale index would score the old way.
+ */
+const INDEX_VERSION = 1;
+
+/**
+ * What the tokenizer does to a probe that exercises its rules: case, camel
+ * and snake splitting, digits, punctuation, stopwords and short words. Part of
+ * the index key, so a change to `bm25Tokens` that shows here invalidates every
+ * stored index even if nobody bumps INDEX_VERSION -- an upgrade must never
+ * score new queries against an old tokenizer's postings.
+ */
+const TOKENIZER_PROBE =
+ "The saveModel() of get_openai_callback fails: JsonWriter.toJsonTree in src/a-b/C.ts line 42, a to is HTTPServer2";
+export const TOKENIZER_FINGERPRINT = createHash("sha256")
+ .update(JSON.stringify(bm25Tokens(TOKENIZER_PROBE))).digest("hex").slice(0, 16);
+
+interface Bm25Index {
+ v: number;
+ /** `TOKENIZER_FINGERPRINT` when built. */
+ tok: string;
+ root: string;
+ head: string;
+ maxBytes: number;
+ /** Clean files that were scored, and each one's length in words. */
+ paths: string[];
+ lengths: number[];
+ /** Clean files that were not scored (too large or unreadable at HEAD). */
+ skipped: string[];
+ /** Every word, to a flat list of [document index, count] pairs. */
+ postings: Record<string, number[]>;
+}
+
+/** What the working tree is, as far as the index is concerned. */
+export interface GitState {
+ head: string;
+ /** Tracked paths (relative to the root, as `git ls-files` prints them) that differ from HEAD. */
+ dirty: ReadonlySet<string>;
+}
+
+/** Path of the index for one workspace root, under IX_HOME. */
+export function bm25IndexPath(root: string): string {
+ const key = createHash("sha256").update(root).digest("hex").slice(0, 12);
+ return join(ixHome(), "cache", `bm25_${key}.json`);
+}
+
+/**
+ * HEAD and the files that differ from it, or undefined outside a checkout (or
+ * on an unborn branch). `--no-optional-locks` because this runs in whatever
+ * checkout the user is in, and a read must not rewrite its index file.
+ */
+export function readGitState(root: string): GitState | undefined {
+ try {
+ const run = (args: string[]) => execFileSync("git", ["--no-optional-locks", ...args], {
+ cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024,
+ });
+ const head = run(["rev-parse", "--verify", "-q", "HEAD"]).trim();
+ if (!head) return undefined;
+ // --relative: paths relative to `root`, like `git ls-files` run there.
+ // --no-renames: a rename lists both of its paths, not only the new one.
+ const dirty = run(["diff", "--name-only", "--relative", "--no-renames", "-z", "HEAD", "--"]);
+ return { head, dirty: new Set(dirty.split("\0").filter(Boolean)) };
+ } catch {
+ return undefined;
+ }
+}
+
+/**
+ * A `planIssue` ranker that reads through the index for `root`. Falls back to
+ * plain `bm25Rank` when there is no git state to key it by.
+ */
+export function cachedBm25Ranker(
+ root: string,
+ opts: { state?: () => GitState | undefined; indexPath?: string } = {},
+): (repo: RepoAccess, files: string[], query: string) => Bm25Hit[] {
+ return (repo, files, query) => {
+ const readState = opts.state ?? (() => readGitState(root));
+ const state = readState();
+ if (!state) return bm25Rank(repo, files, query);
+ const indexPath = opts.indexPath ?? bm25IndexPath(root);
- const terms = bm25QueryTerms(query);
++ const weights = bm25QueryWeights(query);
++ const terms = [...weights.keys()];
+ const index = loadIndex(indexPath, root, state.head);
- if (index) return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), terms);
++ if (index) return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), weights);
+
+ const { docs, built } = buildIndex(repo, files, terms, state, root);
+ // Only if nothing moved while the files were being read: a file edited in
+ // that window would otherwise be stored as HEAD's text.
+ const after = readState();
+ if (after && after.head === state.head && sameSet(after.dirty, state.dirty)) saveIndex(indexPath, built);
- return bm25Score(docs, terms);
++ return bm25Score(docs, weights);
+ };
+}
+
+function docsFromIndex(
+ index: Bm25Index,
+ repo: Pick<RepoAccess, "read">,
+ files: string[],
+ terms: string[],
+ dirty: ReadonlySet<string>,
+): Map<string, Bm25Doc> {
+ const position = new Map(index.paths.map((path, i) => [path, i]));
+ const skipped = new Set(index.skipped);
+ // Only the query's words, as bm25Rank counts them.
+ const tfs: Array<Map<string, number> | undefined> = new Array(index.paths.length);
+ for (const term of terms) {
+ if (!Object.hasOwn(index.postings, term)) continue;
+ const list = index.postings[term];
+ for (let k = 0; k < list.length; k += 2) (tfs[list[k]] ??= new Map()).set(term, list[k + 1]);
+ }
+ const wanted = new Set(terms);
+ const docs = new Map<string, Bm25Doc>();
+ for (const path of files) {
+ if (!dirty.has(path)) {
+ const i = position.get(path);
+ if (i !== undefined) {
+ docs.set(path, { tf: tfs[i] ?? new Map(), length: index.lengths[i] });
+ continue;
+ }
+ if (skipped.has(path)) continue;
+ }
+ // Dirty, or (should not happen at one HEAD) a file the index never saw.
+ const doc = bm25Doc(path, repo.read(path), wanted);
+ if (doc) docs.set(path, doc);
+ }
+ return docs;
+}
+
+function buildIndex(
+ repo: Pick<RepoAccess, "read">,
+ files: string[],
+ terms: string[],
+ state: GitState,
+ root: string,
+): { docs: Map<string, Bm25Doc>; built: Bm25Index } {
+ const built: Bm25Index = {
+ v: INDEX_VERSION, tok: TOKENIZER_FINGERPRINT, root, head: state.head, maxBytes: MAX_BM25_BYTES,
+ paths: [], lengths: [], skipped: [],
+ // No prototype: `constructor` and `tostring` are words too.
+ postings: Object.create(null) as Record<string, number[]>,
+ };
+ const docs = new Map<string, Bm25Doc>();
+ for (const path of files) {
+ const full = bm25Doc(path, repo.read(path));
+ const clean = !state.dirty.has(path);
+ if (!full) {
+ if (clean) built.skipped.push(path);
+ continue;
+ }
+ if (clean) {
+ const i = built.paths.length;
+ built.paths.push(path);
+ built.lengths.push(full.length);
+ for (const [term, count] of full.tf) (built.postings[term] ??= []).push(i, count);
+ }
+ const tf = new Map<string, number>();
+ for (const term of terms) {
+ const count = full.tf.get(term);
+ if (count) tf.set(term, count);
+ }
+ docs.set(path, { tf, length: full.length });
+ }
+ return { docs, built };
+}
+
+function loadIndex(path: string, root: string, head: string): Bm25Index | undefined {
+ try {
+ const data = JSON.parse(readFileSync(path, "utf-8")) as Partial<Bm25Index>;
+ if (data.v !== INDEX_VERSION || data.tok !== TOKENIZER_FINGERPRINT || data.root !== root || data.head !== head || data.maxBytes !== MAX_BM25_BYTES) return undefined;
+ if (!Array.isArray(data.paths) || !Array.isArray(data.lengths) || !Array.isArray(data.skipped)) return undefined;
+ if (data.paths.length !== data.lengths.length || !data.postings || typeof data.postings !== "object") return undefined;
+ return data as Bm25Index;
+ } catch {
+ return undefined;
+ }
+}
+
+/** Best-effort: an index that cannot be written only costs the next call a rebuild. */
+function saveIndex(path: string, index: Bm25Index): void {
+ const tmp = `${path}.${process.pid}.tmp`;
+ try {
+ mkdirSync(dirname(path), { recursive: true });
+ // Via a temp file and rename, so a reader never sees half a file.
+ writeFileSync(tmp, JSON.stringify(index));
+ renameSync(tmp, path);
+ } catch {
+ try { rmSync(tmp, { force: true }); } catch { /* non-critical */ }
+ }
+}
+
+function sameSet(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
+ if (a.size !== b.size) return false;
+ for (const x of a) if (!b.has(x)) return false;
+ return true;
+}
diff --cc ix-cli/src/cli/explain/facts.ts
index 931b5a6b,308041ac..a2a94d86
--- a/ix-cli/src/cli/explain/facts.ts
+++ b/ix-cli/src/cli/explain/facts.ts
@@@ -304,52 -303,6 +305,52 @@@ async function rankMembersByUse(client
return [...ranked, ...structural.slice(MEMBER_USE_POOL)];
}
+/** The entity that contains the target, or undefined when it cannot be read. */
+async function lookUpContainer(client: IxClient, containerId: string): Promise<EntityLocation | undefined> {
+ try {
+ const containerDetails = await client.entity(containerId);
+ return toLocation(containerDetails.node);
+ } catch {
+ return undefined; // no container
+ }
+}
+
+/** The target's callees by name, one entity read each, in edge order. */
+function resolveCallList(client: IxClient, calleeEdges: any[]): Promise<EntityRef[]> {
+ return Promise.all(
+ calleeEdges.map(async (e: any): Promise<EntityRef> => {
+ try {
+ const callee = await client.entity(e.dst);
+ const calleeNode = callee.node as any;
+ const name = calleeNode.name || calleeNode.attrs?.name || "";
+ if (!name || isRawId(name)) {
+ return {
+ name: name || e.dst,
+ kind: calleeNode.kind,
+ resolved: false,
- suggestedCommand: `ix text "${e.dst.slice(0, 8)}"`,
++ suggestedCommand: forMcp() ? toolCall("ix_text", { pattern: e.dst.slice(0, 8) }) : `ix text "${e.dst.slice(0, 8)}"`,
+ };
+ }
+ return {
+ name,
+ kind: calleeNode.kind,
+ id: e.dst,
+ resolved: true,
+ path: relativePath(calleeNode.provenance?.source_uri ?? calleeNode.provenance?.sourceUri),
- suggestedCommand: `ix explain "${name}"`,
++ suggestedCommand: forMcp() ? toolCall("ix_explain", { symbol: name }) : `ix explain "${name}"`,
+ };
+ } catch {
+ return {
+ name: e.dst,
+ resolved: false,
+ diagnostic: "unresolved_call_target",
- suggestedCommand: `ix text "${e.dst.slice(0, 8)}"`,
++ suggestedCommand: forMcp() ? toolCall("ix_text", { pattern: e.dst.slice(0, 8) }) : `ix text "${e.dst.slice(0, 8)}"`,
+ } as EntityRef;
+ }
+ }),
+ );
+}
+
/**
* Collect the structural facts about one entity.
*
diff --cc ix-cli/src/cli/explain/issue.ts
index 2105991c,b4cf9ec1..33b25630
--- a/ix-cli/src/cli/explain/issue.ts
+++ b/ix-cli/src/cli/explain/issue.ts
@@@ -261,35 -301,44 +301,74 @@@ export function bm25Tokens(text: string
return out;
}
+ /**
+ * Words an issue is written in that say nothing about where the code is:
+ * English function words and the vocabulary of bug reports. Dropped from the
+ * query only. Code comments are English too, so without this "should",
+ * "would" and "expected" pull the most-commented files up.
+ */
+ const BM25_STOPWORDS = new Set((
+ "the and for are but not you all any can had her was one our out has him his how its may new now old see "
+ + "two way who did get let say she too use about above after again also been before being below between both "
+ + "could does doing down during each few from further have having here into just more most much must only other "
+ + "over own same should some such than that their them then there these they this those through under until very "
+ + "want what when where which while whom why will with would your yours "
+ + "bug bugs issue issues expected behavior behaviour actual reproduce reproduction steps version versions "
+ + "current currently describe description problem thanks thank please like think seems seem happen happens "
+ + "work works working instead because since using used example following sure able still even though"
+ ).split(" "));
+
+ /**
+ * How much more a query word counts when it is part of a code name the issue
+ * mentions (`extractCandidates`) than when it is only in the prose.
+ */
+ export const BM25_IDENTIFIER_WEIGHT = 2;
+
+ /** Each query word, with its weight: 1 for prose, more for a word of a named identifier. */
-function bm25Query(query: string): Map<string, number> {
++export function bm25QueryWeights(query: string): Map<string, number> {
+ const weights = new Map<string, number>();
+ for (const t of bm25Tokens(query)) if (!BM25_STOPWORDS.has(t)) weights.set(t, 1);
+ for (const id of extractCandidates(query).identifiers) {
+ for (const t of bm25Tokens(id)) if (weights.has(t)) weights.set(t, BM25_IDENTIFIER_WEIGHT);
+ }
+ return weights;
+ }
+
export interface Bm25Hit {
path: string;
score: number;
}
+/** One file as BM25 reads it: how often each counted word occurs, and its length in words. */
+export interface Bm25Doc {
+ tf: Map<string, number>;
+ length: number;
+}
+
+/**
+ * A file's text as BM25 reads it, or undefined when it is not scored (too
+ * large, or unreadable). `wanted` limits the words counted -- see `bm25Rank`;
+ * without it every word is, which is what the on-disk index stores.
+ */
+export function bm25Doc(path: string, text: string | undefined, wanted?: ReadonlySet<string>): Bm25Doc | undefined {
+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES) return undefined;
+ const tf = new Map<string, number>();
- const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);
- for (const t of tokens) if (!wanted || wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);
- return { tf, length: tokens.length };
++ let length = 0;
++ // Counted as the words stream past: no array per file.
++ const count = (t: string) => {
++ length++;
++ if (!wanted || wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);
++ };
++ eachBm25Token(path.replace(/\//g, " "), count);
++ eachBm25Token(text, count);
++ return { tf, length };
+}
+
- /** The query's distinct BM25 words, in the order it first uses them. */
++/** The query's scored words (stopwords dropped), in the order it first uses them. */
+export function bm25QueryTerms(query: string): string[] {
- return [...new Set(bm25Tokens(query))];
++ return [...bm25QueryWeights(query).keys()];
+}
+
/**
* Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files
* that share no word with it left out. A file's path is part of its text, so
@@@ -305,27 -361,22 +391,27 @@@ export function bm25Rank
// Only the query's words are counted: BM25 needs no other term frequency,
// and a map of every word of every file is most of the memory and a third
// of the time on a large repository. A document's length is still every word.
- const terms = bm25QueryTerms(query);
- const wanted = new Set(terms);
- const weights = bm25Query(query);
- const terms = [...weights.keys()];
- const docs = new Map<string, { tf: Map<string, number>; length: number }>();
++ const weights = bm25QueryWeights(query);
++ const wanted = new Set(weights.keys());
+ const docs = new Map<string, Bm25Doc>();
for (const path of files) {
- const text = repo.read(path);
- if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES) continue;
- const tf = new Map<string, number>();
- let length = 0;
- const count = (t: string) => {
- length++;
- if (weights.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);
- };
- eachBm25Token(path.replace(/\//g, " "), count);
- eachBm25Token(text, count);
- docs.set(path, { tf, length });
+ const doc = bm25Doc(path, repo.read(path), wanted);
+ if (doc) docs.set(path, doc);
}
- return bm25Score(docs, terms, k1, b);
++ return bm25Score(docs, weights, k1, b);
+}
+
+/**
+ * BM25 of already-tokenized files. Split out of `bm25Rank` so the on-disk
+ * index (`bm25-cache.ts`) scores through the very same arithmetic: the same
+ * documents in, the same scores out, to the last bit.
+ */
+export function bm25Score(
+ docs: ReadonlyMap<string, Bm25Doc>,
- terms: string[],
++ weights: ReadonlyMap<string, number>,
+ k1 = 1.2,
+ b = 0.75,
+): Bm25Hit[] {
if (docs.size === 0) return [];
const n = docs.size;
let total = 0;
@@@ -338,12 -389,12 +424,12 @@@
const out: Bm25Hit[] = [];
for (const [path, doc] of docs) {
let score = 0;
-- for (const t of terms) {
++ for (const [t, weight] of weights) {
const f = doc.tf.get(t);
if (!f) continue;
const d = df.get(t)!;
const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));
- score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));
- score += weights.get(t)! * idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * doc.length) / avg)));
++ score += weight * idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * doc.length) / avg)));
}
if (score > 0) out.push({ path, score });
}
diff --cc ix-cli/src/client/api.ts
index fb85eb10,c19077a7..c1b54b14
--- a/ix-cli/src/client/api.ts
+++ b/ix-cli/src/client/api.ts
@@@ -691,51 -662,32 +703,51 @@@ export class IxClient
}
private async post<T>(path: string, body: unknown): Promise<T> {
- const resp = await fetch(`${this.endpoint}${path}`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify(body),
- // These small reads/writes (source-hashes, stitch, list, ...) had no
- // timeout, so a stalled connection could hang the process indefinitely.
- // 2 min per request, also bounded by the shared deadline when set.
- signal: this.signalFor(2 * 60 * 1000),
- });
- if (!resp.ok) {
- const text = await resp.text();
- throw new Error(`${resp.status}: ${text}`);
- }
- return parseOrThrowWithStatus<T>(resp);
+ const payload = JSON.stringify(body);
+ return this.read<T>("POST", path, payload, () =>
+ fetch(`${this.endpoint}${path}`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: payload,
+ // These small reads/writes (source-hashes, stitch, list, ...) had no
+ // timeout, so a stalled connection could hang the process indefinitely.
+ // 2 min per request, also bounded by the shared deadline when set.
+ signal: this.signalFor(2 * 60 * 1000),
+ }));
}
- private async get<T>(path: string): Promise<T> {
+ private async get<T>(path: string, timeoutMs = 2 * 60 * 1000): Promise<T> {
- const resp = await fetch(`${this.endpoint}${path}`, {
- signal: this.signalFor(timeoutMs),
- });
- if (!resp.ok) {
- const text = await resp.text();
- throw new Error(`${resp.status}: ${text}`);
- }
- return parseOrThrowWithStatus<T>(resp);
+ return this.read<T>("GET", path, "", () =>
+ fetch(`${this.endpoint}${path}`, {
- signal: this.signalFor(2 * 60 * 1000),
++ signal: this.signalFor(timeoutMs),
+ }));
}
+
+ /**
+ * One request whose body is JSON: sent under the in-flight limit when there
+ * is one, shared with an identical earlier request when `shareReads` is on
+ * and the path is a read, and parsed per caller either way.
+ */
+ private async read<T>(method: string, path: string, payload: string, send: () => Promise<Response>): Promise<T> {
+ const fetchBody = async (): Promise<SuccessBody> => {
+ const run = async (): Promise<SuccessBody> => {
+ const resp = await send();
+ const text = await resp.text();
+ if (!resp.ok) throw new Error(`${resp.status}: ${text}`);
+ return { status: resp.status, text };
+ };
+ return this.limiter ? this.limiter.run(run) : run();
+ };
+ const body = this.memo && isSharedRead(path)
+ ? await this.memo.run(`${method} ${path} ${payload}`, fetchBody)
+ : await fetchBody();
+ return parseOrThrowWithStatus<T>(body);
+ }
+}
+
+interface SuccessBody {
+ status: number;
+ text: string;
}
/**