Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
Merge origin/main into perf/query-paths
KageBinary6d708bedee79
diff --cc ix-cli/src/cli/commands/context.tsindex 7200f908,ded9180c..aeeaf67f--- a/ix-cli/src/cli/commands/context.ts+++ b/ix-cli/src/cli/commands/context.ts@@@ -1632,32 -1671,10 +1673,10 @@@ async function buildIssueBundleopts: ContextOptions,budgets: BudgetSnapshot,): Promise<ContextBundle | undefined> {- let text: string;- try {- text = await readIssueText(arg);- } catch (error) {- issueFailure(- "issue_unreadable",- `Cannot read the issue from ${arg === "-" ? "stdin" : `"${arg}"`}: ${(error as Error).message}`,- opts.format,- );- return undefined;- }- if (!text.trim()) {- issueFailure("empty_issue", `The issue ${arg === "-" ? "on stdin" : `in "${arg}"`} is empty.`, opts.format);- return undefined;- }-+ const text = await readIssueOrReport(arg, opts.format);+ if (text === undefined) return undefined;- const client = new IxClient(getEndpoint());+ const client = new IxClient(getEndpoint(), undefined, QUERY_CLIENT_OPTIONS);- await ensureReadScope(client);- const scope = activeReadScope();- const root = resolveWorkspaceRoot();- const plan = await planIssue(text, {- repo: gitRepoAccess(root),- rank: cachedBm25Ranker(root),- search: async (name) =>- (await client.search(name, { limit: ISSUE_SEARCH_LIMIT, nameOnly: true, ...scope })).map(symbolHit),- });+ const plan = await planIssueWith(client, text);// A path start has no node yet, and the graph may not have one either: it// does not index every tracked file. chooseCentre walks BM25 for one it does.const { starts, centre, walked } = await chooseCentre(@@@ -1711,6 -1730,95 +1732,99 @@@});}+ /** The issue's text, or undefined after reporting why it has none. */+ async function readIssueOrReport(arg: string, format: string | undefined): Promise<string | undefined> {+ let text: string;+ try {+ text = await readIssueText(arg);+ } catch (error) {+ issueFailure(+ "issue_unreadable",+ `Cannot read the issue from ${arg === "-" ? "stdin" : `"${arg}"`}: ${(error as Error).message}`,+ format,+ );+ return undefined;+ }+ if (!text.trim()) {+ issueFailure("empty_issue", `The issue ${arg === "-" ? "on stdin" : `in "${arg}"`} is empty.`, format);+ return undefined;+ }+ return text;+ }++ /** The issue's starting points and lexical ranking, against this workspace's graph. */+ async function planIssueWith(client: IxClient, text: string) {+ await ensureReadScope(client);+ const scope = activeReadScope();++ const root = resolveWorkspaceRoot();+ return planIssue(text, {- repo: gitRepoAccess(resolveWorkspaceRoot()),++ repo: gitRepoAccess(root),++ // Through the on-disk term index (bm25-cache.ts): the same ranking, with++ // only the files that differ from HEAD read again.++ rank: cachedBm25Ranker(root),+ search: async (name) =>+ (await client.search(name, { limit: ISSUE_SEARCH_LIMIT, nameOnly: true, ...scope })).map(symbolHit),+ });+ }++ /**+ * `ix context --from-issue --lean`: the starting points Ix trusts and the+ * next few files, with no bundle around them. See `leanIssueView` for why.+ * Nothing here needs a centre node, so it is also the cheap path: one search+ * per name the issue mentions, and BM25 over the tracked files.+ */+ async function emitLeanIssue(arg: string, opts: ContextOptions): Promise<void> {+ const text = await readIssueOrReport(arg, opts.format);+ if (text === undefined) return;- const view = leanIssueView(await planIssueWith(new IxClient(getEndpoint()), text));++ const view = leanIssueView(await planIssueWith(new IxClient(getEndpoint(), undefined, QUERY_CLIENT_OPTIONS), text));+ if (opts.format === "json") {+ printJson({ kind: "issue_lean", ...view });+ } else if (opts.format === "llm") {+ printLlmLines(renderLeanIssueLlm(view));+ } else {+ console.log(renderLeanIssueText(view));+ }+ }++ function spanOf(s: StartingPoint): string {+ return s.lineStart !== undefined ? `${s.path}:${s.lineStart}-${s.lineEnd ?? s.lineStart}` : s.path;+ }++ /** Why a start was trusted, in the words an agent reads. */+ function startWhy(s: StartingPoint): string {+ return s.via === "path in issue" ? "the issue names this file" : `the issue names \`${s.token}\``;+ }++ export function renderLeanIssueText(view: LeanIssueView): string {+ if (view.confidence === "low") {+ return `Ix found no confident starting point for this issue: ${view.reason}.`;+ }+ const lines = ["Where to start (Ix resolved these from the issue's own words):"];+ for (const s of view.startingPoints) {+ lines.push(` ${spanOf(s)} ${s.kind === "file" ? "" : `${s.name} (${s.kind}) — `}${startWhy(s)}`);+ }+ if (view.alsoRanked.length > 0) {+ lines.push(`Then, by the issue's text: ${view.alsoRanked.map((f) => f.path).join(", ")}`);+ }+ return lines.join("\n");+ }++ export function renderLeanIssueLlm(view: LeanIssueView): string[] {+ const out = [llmLine("issue", [+ ["confidence", view.confidence],+ ...(view.reason ? [["reason", view.reason] as [string, string]] : []),+ ])];+ for (const s of view.startingPoints) {+ out.push(llmLine("start", [+ ["path", s.path],+ ...(s.lineStart !== undefined ? [["lines", `${s.lineStart}-${s.lineEnd ?? s.lineStart}`] as [string, string]] : []),+ ["name", s.name], ["kind", s.kind], ["why", startWhy(s)],+ ]));+ }+ for (const f of view.alsoRanked) out.push(llmLine("ranked", [["path", f.path]]));+ return out;+ }+/*** A `--from-issue` failure, readable by the format asked for. `reportFailure`* writes nothing to stdout under `--format json`, so a JSON caller got an emptydiff --cc ix-cli/src/cli/commands/explain.tsindex c254db3a,4fc9730f..589ef666--- a/ix-cli/src/cli/commands/explain.ts+++ b/ix-cli/src/cli/commands/explain.ts@@@ -2,10 -2,9 +2,10 @@@import type { Command } from "commander";import { IxClient } from "../../client/api.js";+import { QUERY_CLIENT_OPTIONS } from "../../client/request-memo.js";import { getEndpoint } from "../config.js";import { formatExplain, relativePath, printJson, type ExplainResult, type EntityRef, type Diagnostic } from "../format.js";- import { resolveFileOrReport, isRawId } from "../resolve.js";+ import { resolveFileOrReport, isRawId, activeReadScope } from "../resolve.js";import { isFileStale } from "../stale.js";import { collectFacts } from "../explain/facts.js";import { inferRole } from "../explain/role-inference.js";diff --cc ix-cli/src/cli/commands/impact.tsindex a6ad1c8a,5af5eade..756568b3--- a/ix-cli/src/cli/commands/impact.ts+++ b/ix-cli/src/cli/commands/impact.ts@@@ -2,14 -2,14 +2,15 @@@import type { Command } from "commander";import chalk from "chalk";- import { renderSection, renderKeyValue, renderNote, renderResolvedHeader, colorizeKind } from "../ui.js";+ import { renderSection, renderKeyValue, renderNote, renderResolvedHeader, renderWarning, colorizeKind } from "../ui.js";import { IxClient } from "../../client/api.js";+import { QUERY_CLIENT_OPTIONS } from "../../client/request-memo.js";import { getEndpoint } from "../config.js";- import { resolveFileOrReport, printResolved } from "../resolve.js";+ import { activeReadScope, resolveFileOrReport, printResolved } from "../resolve.js";+ import { checkGraphHealth, graphHealthJson, graphHealthLlmFields, graphHealthProse, isUnhealthy } from "../graph-health.js";import { bucketByHierarchy, getSystemPath, formatSystemPath, hasMapData, type SystemPath } from "../hierarchy.js";import { inferRiskSemantics, humanizeLabel, type ImpactFacts, type RiskSemantics } from "../impact/risk-semantics.js";- import { lineSpan, printJson, rowLocation, stripNulls } from "../format.js";+ import { lineSpan, printJson, relativePath, rowLocation, stripNulls } from "../format.js";import { llmLine } from "../llm.js";import { parsePickOption } from "../options.js";diff --cc ix-cli/src/cli/context-bundle-schema.tsindex 85e74e29,6f3be6f1..ba7d85d0--- a/ix-cli/src/cli/context-bundle-schema.ts+++ b/ix-cli/src/cli/context-bundle-schema.ts@@@ -51,170 -74,125 +51,182 @@@ function buildSchemas(z: typeof Zod.z)id: z.string(),name: z.string(),kind: z.string(),+ resolutionMode: z.string(),path: z.string().optional(),- lineStart: z.number().int().positive().optional(),- lineEnd: z.number().int().positive().optional(),- stale: z.boolean(),}),- ),- relationships: z.array(z.object({ src: z.string(), dst: z.string(), predicate: z.string() })),- claims: z.array(- z- .object({ id: z.string(), entityId: z.string(), statement: z.string(), status: z.string() })+ // Both optional, and only on a bundle built with `--from-issue`: every+ // bundle saved before them, and every bundle built from a named target,+ // still validates. Declared rather than left out, because zod strips an+ // undeclared key on the way to disk and on the way back.+ issue: z+ .object({+ startingPoints: z.array(+ z.object({+ token: z.string(),+ id: z.string().optional(),+ name: z.string(),+ kind: z.string(),+ path: z.string(),+ lineStart: z.number().int().positive().optional(),+ lineEnd: z.number().int().positive().optional(),+ via: z.enum(["path in issue", "identifier in issue", "bm25 fallback"]),+ }),+ ),+ unresolved: z.array(z.string()),+ fallback: z.boolean(),+ })+ .optional(),+ rankedFiles: z+ .array(z.object({ path: z.string(), score: z.number(), reason: z.string() }))+ .optional(),+ entities: z.array(+ z.object({+ id: z.string(),+ name: z.string(),+ kind: z.string(),+ path: z.string().optional(),+ lineStart: z.number().int().positive().optional(),+ lineEnd: z.number().int().positive().optional(),+ stale: z.boolean(),+ }),+ ),+ relationships: z.array(z.object({ src: z.string(), dst: z.string(), predicate: z.string() })),+ claims: z.array(+ z+ .object({ id: z.string(), entityId: z.string(), statement: z.string(), status: z.string() })+ .catchall(z.unknown()),+ ),+ // DecisionReport/ConflictReport/IntentReport are the backend's own contracts+ // with their own shapes; they are forwarded, never dereferenced field-by-field+ // here, so they stay loose rather than duplicating those types in a second+ // place that could drift from them.+ decisions: z.array(z.record(z.string(), z.unknown())),+ conflicts: z.array(z.record(z.string(), z.unknown())),+ intents: z.array(z.record(z.string(), z.unknown())),+ provenance: z+ .object({+ sourceUri: z.string().optional(),+ sourceHash: z.string().optional(),+ extractor: z.string().optional(),+ sourceType: z.string().optional(),+ observedAt: z.string().optional(),+ introducedRev: z.number().optional(),+ historyLength: z.number(),+ stale: z.boolean(),+ }).catchall(z.unknown()),- ),- // DecisionReport/ConflictReport/IntentReport are the backend's own contracts- // with their own shapes; they are forwarded, never dereferenced field-by-field- // here, so they stay loose rather than duplicating those types in a second- // place that could drift from them.- decisions: z.array(z.record(z.string(), z.unknown())),- conflicts: z.array(z.record(z.string(), z.unknown())),- intents: z.array(z.record(z.string(), z.unknown())),- provenance: z- .object({- sourceUri: z.string().optional(),- sourceHash: z.string().optional(),- extractor: z.string().optional(),- sourceType: z.string().optional(),- observedAt: z.string().optional(),- introducedRev: z.number().optional(),- historyLength: z.number(),- stale: z.boolean(),- })- .catchall(z.unknown()),- freshness: z.object({ stale: z.boolean(), classification: z.string() }),- // Only when the graph the bundle was drawn from is degraded or empty (see- // graph-health.ts). Optional and additive, like `issue`; declared so zod- // does not strip it on the way to disk.- graph: z- .object({- status: z.string(),- reason: z.string().optional(),- message: z.string().optional(),- fix: z.string().optional(),- })- .catchall(z.unknown())- .optional(),- evidence: z.array(- z.object({- id: z.string(),- kind: z.string(),- source: z.string(),- title: z.string(),- score: z.number(),- reason: z.string(),- refs: z.array(z.string()),- // Optional and additive: bundles saved before it existed still parse.- // Listed rather than left to `catchall`, because an unlisted key is- // stripped on the way to disk and on the way back.- location: z- .object({- path: z.string(),- lineStart: z.number().int().positive().optional(),- lineEnd: z.number().int().positive().optional(),- })- .optional(),+ freshness: z.object({ stale: z.boolean(), classification: z.string() }),++ // Only when the graph the bundle was drawn from is degraded or empty (see++ // graph-health.ts). Optional and additive, like `issue`; declared so zod++ // does not strip it on the way to disk.++ graph: z++ .object({++ status: z.string(),++ reason: z.string().optional(),++ message: z.string().optional(),++ fix: z.string().optional(),++ })++ .catchall(z.unknown())++ .optional(),+ evidence: z.array(+ z.object({+ id: z.string(),+ kind: z.string(),+ source: z.string(),+ title: z.string(),+ score: z.number(),+ reason: z.string(),+ refs: z.array(z.string()),+ // Optional and additive: bundles saved before it existed still parse.+ // Listed rather than left to `catchall`, because an unlisted key is+ // stripped on the way to disk and on the way back.+ location: z+ .object({+ path: z.string(),+ lineStart: z.number().int().positive().optional(),+ lineEnd: z.number().int().positive().optional(),+ })+ .optional(),+ }),+ ),+ budgets: z.object({+ maxEntities: z.number().int().positive(),+ maxRelationships: z.number().int().positive(),+ maxEvidence: z.number().int().positive(),+ // Optional, and only this one. A bundle built before `--max-tokens`+ // existed carries the other four and nothing else, and zod strips keys it+ // does not declare — so leaving it out here would not have been "tolerant+ // of old files", it would have silently dropped the field from every new+ // one on its way to disk. Required would refuse to load those old files+ // instead, which is a worse answer than reporting `tokens=not-given`.+ maxTokens: z.number().int().positive().optional(),+ maxChars: z.number().int().positive(),}),- ),- budgets: z.object({- maxEntities: z.number().int().positive(),- maxRelationships: z.number().int().positive(),- maxEvidence: z.number().int().positive(),- // Optional, and only this one. A bundle built before `--max-tokens`- // existed carries the other four and nothing else, and zod strips keys it- // does not declare — so leaving it out here would not have been "tolerant- // of old files", it would have silently dropped the field from every new- // one on its way to disk. Required would refuse to load those old files- // instead, which is a worse answer than reporting `tokens=not-given`.- maxTokens: z.number().int().positive().optional(),- maxChars: z.number().int().positive(),- }),- truncation: z.object({- // Optional for the same reason `budgets.maxTokens` is: a bundle saved- // before this field existed carries the counters and nothing else, and an- // undeclared key would be stripped from every new one on its way to disk.- cut: z.array(z.object({ what: z.string(), count: z.number().int().positive() })).optional(),- entitiesTruncated: z.number().int().nonnegative(),- relationshipsTruncated: z.number().int().nonnegative(),- evidenceTruncated: z.number().int().nonnegative(),- charactersTruncated: z.number().int().nonnegative(),- // Optional on the way in, always written on the way out: a bundle saved- // before the conflict cap existed has no such counter, and refusing to- // read it back would break `--diff` against every investigation on disk.- conflictsTruncated: z.number().int().nonnegative().optional(),- }),- // `asOfRev` and `depth` are the two saved values `ix context --diff` re-sends- // to the backend, so they are typed rather than accepted as anything.- // Unknown keys pass through: metadata is the bundle's extension point, and- // stripping them would silently drop data on the read-back path.- metadata: z- .object({- asOfRev: z.number().optional(),- depth: z.string().optional(),- rankingRule: z.string(),- })- .catchall(z.unknown()),-});+ truncation: z.object({+ // Optional for the same reason `budgets.maxTokens` is: a bundle saved+ // before this field existed carries the counters and nothing else, and an+ // undeclared key would be stripped from every new one on its way to disk.+ cut: z.array(z.object({ what: z.string(), count: z.number().int().positive() })).optional(),+ entitiesTruncated: z.number().int().nonnegative(),+ relationshipsTruncated: z.number().int().nonnegative(),+ evidenceTruncated: z.number().int().nonnegative(),+ charactersTruncated: z.number().int().nonnegative(),+ // Optional on the way in, always written on the way out: a bundle saved+ // before the conflict cap existed has no such counter, and refusing to+ // read it back would break `--diff` against every investigation on disk.+ conflictsTruncated: z.number().int().nonnegative().optional(),+ }),+ // `asOfRev` and `depth` are the two saved values `ix context --diff` re-sends+ // to the backend, so they are typed rather than accepted as anything.+ // Unknown keys pass through: metadata is the bundle's extension point, and+ // stripping them would silently drop data on the read-back path.+ metadata: z+ .object({+ asOfRev: z.number().optional(),+ depth: z.string().optional(),+ rankingRule: z.string(),+ })+ .catchall(z.unknown()),+ });-/**- * Contract for the on-disk saved-investigation envelope (`~/.ix/investigations`).- *- * `id` and `savedAt` are rendered to the terminal and copied into the emitted- * `ix-investigation-diff/1` JSON, so they are validated here alongside the- * bundle rather than trusted from a bare cast.- */-export const savedInvestigationSchema = z.object({- schema: z.literal(INVESTIGATION_SCHEMA),- // `sanitizeId` percent-escapes anything outside this set on the way in, so a- // stored id can only contain these characters. Pinning the same set on the- // way out keeps control characters and ANSI escapes out of an id that- // `renderNote`/`renderSection` write straight to the terminal.- id: z.string().regex(/^[A-Za-z0-9._~-]+$/),- savedAt: z.string(),- bundle: contextBundleSchema,-});+ /**+ * Contract for the on-disk saved-investigation envelope (`~/.ix/investigations`).+ *+ * `id` and `savedAt` are rendered to the terminal and copied into the emitted+ * `ix-investigation-diff/1` JSON, so they are validated here alongside the+ * bundle rather than trusted from a bare cast.+ */+ const savedInvestigationSchema = z.object({+ schema: z.literal(INVESTIGATION_SCHEMA),+ // `sanitizeId` percent-escapes anything outside this set on the way in, so a+ // stored id can only contain these characters. Pinning the same set on the+ // way out keeps control characters and ANSI escapes out of an id that+ // `renderNote`/`renderSection` write straight to the terminal.+ id: z.string().regex(/^[A-Za-z0-9._~-]+$/),+ savedAt: z.string(),+ bundle: contextBundleSchema,+ });++ return { contextBundleSchema, savedInvestigationSchema };+}++let built: ReturnType<typeof buildSchemas> | undefined;+function schemas(): ReturnType<typeof buildSchemas> {+ return (built ??= buildSchemas((requireZod("zod") as typeof Zod).z));+}++type BundleSchema = ReturnType<typeof buildSchemas>["contextBundleSchema"];+type InvestigationSchema = ReturnType<typeof buildSchemas>["savedInvestigationSchema"];++/** The bundle contract (see `buildSchemas`); zod is loaded on the first parse. */+export const contextBundleSchema = {+ parse: (value: unknown): Zod.infer<BundleSchema> => schemas().contextBundleSchema.parse(value),+ safeParse: (value: unknown): ReturnType<BundleSchema["safeParse"]> => schemas().contextBundleSchema.safeParse(value),+};++/** The saved-investigation contract (see `buildSchemas`); zod is loaded on the first parse. */+export const savedInvestigationSchema = {+ parse: (value: unknown): Zod.infer<InvestigationSchema> => schemas().savedInvestigationSchema.parse(value),+ safeParse: (value: unknown): ReturnType<InvestigationSchema["safeParse"]> =>+ schemas().savedInvestigationSchema.safeParse(value),+};diff --cc ix-cli/src/cli/explain/bm25-cache.tsindex 1ed30486,00000000..11a0f099mode 100644,000000..100644--- a/ix-cli/src/cli/explain/bm25-cache.ts+++ b/ix-cli/src/cli/explain/bm25-cache.ts@@@ -1,230 -1,0 +1,231 @@@+// Copyright 2026 Ix Infrastructure Inc.++import { execFileSync } from "node:child_process";+import { createHash } from "node:crypto";+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";+import { dirname, join } from "node:path";++import { ixHome } from "../ix-home.js";+import {+ bm25Doc,- bm25QueryTerms,++ bm25QueryWeights,+ bm25Rank,+ bm25Score,+ bm25Tokens,+ MAX_BM25_BYTES,+ type Bm25Doc,+ type Bm25Hit,+} from "./issue.js";+import type { RepoAccess } from "./text-references.js";++/**+ * A term index of the tracked source files, so `ix context --from-issue` does+ * not read and tokenize every one of them on every call.+ *+ * Without it BM25 opens every tracked source file (up to 200 KB each) per+ * issue: measured at 56 ms on the Ix repository and ~400 ms on apache/dubbo+ * (2,427 files), paid again for the next issue against the same checkout.+ *+ * Keyed by the commit, so the index describes the files AS COMMITTED: it is+ * built from the working tree, but only files that match HEAD are stored, and+ * files that differ from HEAD (`git diff HEAD`, staged or not) are always read+ * fresh. Git decides what is dirty, including the racily-clean case a size and+ * mtime check would miss, so a hit scores exactly the text a fresh read would.+ * One index per workspace root, replaced when HEAD moves.+ *+ * Scores are bit-identical to `bm25Rank`: the same documents go through the+ * same `bm25Score`. Bump INDEX_VERSION whenever `bm25Tokens` or `bm25Doc`+ * changes what a document is, or a stale index would score the old way.+ */+const INDEX_VERSION = 1;++/**+ * What the tokenizer does to a probe that exercises its rules: case, camel+ * and snake splitting, digits, punctuation, stopwords and short words. Part of+ * the index key, so a change to `bm25Tokens` that shows here invalidates every+ * stored index even if nobody bumps INDEX_VERSION -- an upgrade must never+ * score new queries against an old tokenizer's postings.+ */+const TOKENIZER_PROBE =+ "The saveModel() of get_openai_callback fails: JsonWriter.toJsonTree in src/a-b/C.ts line 42, a to is HTTPServer2";+export const TOKENIZER_FINGERPRINT = createHash("sha256")+ .update(JSON.stringify(bm25Tokens(TOKENIZER_PROBE))).digest("hex").slice(0, 16);++interface Bm25Index {+ v: number;+ /** `TOKENIZER_FINGERPRINT` when built. */+ tok: string;+ root: string;+ head: string;+ maxBytes: number;+ /** Clean files that were scored, and each one's length in words. */+ paths: string[];+ lengths: number[];+ /** Clean files that were not scored (too large or unreadable at HEAD). */+ skipped: string[];+ /** Every word, to a flat list of [document index, count] pairs. */+ postings: Record<string, number[]>;+}++/** What the working tree is, as far as the index is concerned. */+export interface GitState {+ head: string;+ /** Tracked paths (relative to the root, as `git ls-files` prints them) that differ from HEAD. */+ dirty: ReadonlySet<string>;+}++/** Path of the index for one workspace root, under IX_HOME. */+export function bm25IndexPath(root: string): string {+ const key = createHash("sha256").update(root).digest("hex").slice(0, 12);+ return join(ixHome(), "cache", `bm25_${key}.json`);+}++/**+ * HEAD and the files that differ from it, or undefined outside a checkout (or+ * on an unborn branch). `--no-optional-locks` because this runs in whatever+ * checkout the user is in, and a read must not rewrite its index file.+ */+export function readGitState(root: string): GitState | undefined {+ try {+ const run = (args: string[]) => execFileSync("git", ["--no-optional-locks", ...args], {+ cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024,+ });+ const head = run(["rev-parse", "--verify", "-q", "HEAD"]).trim();+ if (!head) return undefined;+ // --relative: paths relative to `root`, like `git ls-files` run there.+ // --no-renames: a rename lists both of its paths, not only the new one.+ const dirty = run(["diff", "--name-only", "--relative", "--no-renames", "-z", "HEAD", "--"]);+ return { head, dirty: new Set(dirty.split("\0").filter(Boolean)) };+ } catch {+ return undefined;+ }+}++/**+ * A `planIssue` ranker that reads through the index for `root`. Falls back to+ * plain `bm25Rank` when there is no git state to key it by.+ */+export function cachedBm25Ranker(+ root: string,+ opts: { state?: () => GitState | undefined; indexPath?: string } = {},+): (repo: RepoAccess, files: string[], query: string) => Bm25Hit[] {+ return (repo, files, query) => {+ const readState = opts.state ?? (() => readGitState(root));+ const state = readState();+ if (!state) return bm25Rank(repo, files, query);+ const indexPath = opts.indexPath ?? bm25IndexPath(root);- const terms = bm25QueryTerms(query);++ const weights = bm25QueryWeights(query);++ const terms = [...weights.keys()];+ const index = loadIndex(indexPath, root, state.head);- if (index) return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), terms);++ if (index) return bm25Score(docsFromIndex(index, repo, files, terms, state.dirty), weights);++ const { docs, built } = buildIndex(repo, files, terms, state, root);+ // Only if nothing moved while the files were being read: a file edited in+ // that window would otherwise be stored as HEAD's text.+ const after = readState();+ if (after && after.head === state.head && sameSet(after.dirty, state.dirty)) saveIndex(indexPath, built);- return bm25Score(docs, terms);++ return bm25Score(docs, weights);+ };+}++function docsFromIndex(+ index: Bm25Index,+ repo: Pick<RepoAccess, "read">,+ files: string[],+ terms: string[],+ dirty: ReadonlySet<string>,+): Map<string, Bm25Doc> {+ const position = new Map(index.paths.map((path, i) => [path, i]));+ const skipped = new Set(index.skipped);+ // Only the query's words, as bm25Rank counts them.+ const tfs: Array<Map<string, number> | undefined> = new Array(index.paths.length);+ for (const term of terms) {+ if (!Object.hasOwn(index.postings, term)) continue;+ const list = index.postings[term];+ for (let k = 0; k < list.length; k += 2) (tfs[list[k]] ??= new Map()).set(term, list[k + 1]);+ }+ const wanted = new Set(terms);+ const docs = new Map<string, Bm25Doc>();+ for (const path of files) {+ if (!dirty.has(path)) {+ const i = position.get(path);+ if (i !== undefined) {+ docs.set(path, { tf: tfs[i] ?? new Map(), length: index.lengths[i] });+ continue;+ }+ if (skipped.has(path)) continue;+ }+ // Dirty, or (should not happen at one HEAD) a file the index never saw.+ const doc = bm25Doc(path, repo.read(path), wanted);+ if (doc) docs.set(path, doc);+ }+ return docs;+}++function buildIndex(+ repo: Pick<RepoAccess, "read">,+ files: string[],+ terms: string[],+ state: GitState,+ root: string,+): { docs: Map<string, Bm25Doc>; built: Bm25Index } {+ const built: Bm25Index = {+ v: INDEX_VERSION, tok: TOKENIZER_FINGERPRINT, root, head: state.head, maxBytes: MAX_BM25_BYTES,+ paths: [], lengths: [], skipped: [],+ // No prototype: `constructor` and `tostring` are words too.+ postings: Object.create(null) as Record<string, number[]>,+ };+ const docs = new Map<string, Bm25Doc>();+ for (const path of files) {+ const full = bm25Doc(path, repo.read(path));+ const clean = !state.dirty.has(path);+ if (!full) {+ if (clean) built.skipped.push(path);+ continue;+ }+ if (clean) {+ const i = built.paths.length;+ built.paths.push(path);+ built.lengths.push(full.length);+ for (const [term, count] of full.tf) (built.postings[term] ??= []).push(i, count);+ }+ const tf = new Map<string, number>();+ for (const term of terms) {+ const count = full.tf.get(term);+ if (count) tf.set(term, count);+ }+ docs.set(path, { tf, length: full.length });+ }+ return { docs, built };+}++function loadIndex(path: string, root: string, head: string): Bm25Index | undefined {+ try {+ const data = JSON.parse(readFileSync(path, "utf-8")) as Partial<Bm25Index>;+ if (data.v !== INDEX_VERSION || data.tok !== TOKENIZER_FINGERPRINT || data.root !== root || data.head !== head || data.maxBytes !== MAX_BM25_BYTES) return undefined;+ if (!Array.isArray(data.paths) || !Array.isArray(data.lengths) || !Array.isArray(data.skipped)) return undefined;+ if (data.paths.length !== data.lengths.length || !data.postings || typeof data.postings !== "object") return undefined;+ return data as Bm25Index;+ } catch {+ return undefined;+ }+}++/** Best-effort: an index that cannot be written only costs the next call a rebuild. */+function saveIndex(path: string, index: Bm25Index): void {+ const tmp = `${path}.${process.pid}.tmp`;+ try {+ mkdirSync(dirname(path), { recursive: true });+ // Via a temp file and rename, so a reader never sees half a file.+ writeFileSync(tmp, JSON.stringify(index));+ renameSync(tmp, path);+ } catch {+ try { rmSync(tmp, { force: true }); } catch { /* non-critical */ }+ }+}++function sameSet(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {+ if (a.size !== b.size) return false;+ for (const x of a) if (!b.has(x)) return false;+ return true;+}diff --cc ix-cli/src/cli/explain/facts.tsindex 931b5a6b,308041ac..a2a94d86--- a/ix-cli/src/cli/explain/facts.ts+++ b/ix-cli/src/cli/explain/facts.ts@@@ -304,52 -303,6 +305,52 @@@ async function rankMembersByUse(clientreturn [...ranked, ...structural.slice(MEMBER_USE_POOL)];}+/** The entity that contains the target, or undefined when it cannot be read. */+async function lookUpContainer(client: IxClient, containerId: string): Promise<EntityLocation | undefined> {+ try {+ const containerDetails = await client.entity(containerId);+ return toLocation(containerDetails.node);+ } catch {+ return undefined; // no container+ }+}++/** The target's callees by name, one entity read each, in edge order. */+function resolveCallList(client: IxClient, calleeEdges: any[]): Promise<EntityRef[]> {+ return Promise.all(+ calleeEdges.map(async (e: any): Promise<EntityRef> => {+ try {+ const callee = await client.entity(e.dst);+ const calleeNode = callee.node as any;+ const name = calleeNode.name || calleeNode.attrs?.name || "";+ if (!name || isRawId(name)) {+ return {+ name: name || e.dst,+ kind: calleeNode.kind,+ resolved: false,- suggestedCommand: `ix text "${e.dst.slice(0, 8)}"`,++ suggestedCommand: forMcp() ? toolCall("ix_text", { pattern: e.dst.slice(0, 8) }) : `ix text "${e.dst.slice(0, 8)}"`,+ };+ }+ return {+ name,+ kind: calleeNode.kind,+ id: e.dst,+ resolved: true,+ path: relativePath(calleeNode.provenance?.source_uri ?? calleeNode.provenance?.sourceUri),- suggestedCommand: `ix explain "${name}"`,++ suggestedCommand: forMcp() ? toolCall("ix_explain", { symbol: name }) : `ix explain "${name}"`,+ };+ } catch {+ return {+ name: e.dst,+ resolved: false,+ diagnostic: "unresolved_call_target",- suggestedCommand: `ix text "${e.dst.slice(0, 8)}"`,++ suggestedCommand: forMcp() ? toolCall("ix_text", { pattern: e.dst.slice(0, 8) }) : `ix text "${e.dst.slice(0, 8)}"`,+ } as EntityRef;+ }+ }),+ );+}+/*** Collect the structural facts about one entity.*diff --cc ix-cli/src/cli/explain/issue.tsindex 2105991c,b4cf9ec1..33b25630--- a/ix-cli/src/cli/explain/issue.ts+++ b/ix-cli/src/cli/explain/issue.ts@@@ -261,35 -301,44 +301,74 @@@ export function bm25Tokens(text: stringreturn out;}+ /**+ * Words an issue is written in that say nothing about where the code is:+ * English function words and the vocabulary of bug reports. Dropped from the+ * query only. Code comments are English too, so without this "should",+ * "would" and "expected" pull the most-commented files up.+ */+ const BM25_STOPWORDS = new Set((+ "the and for are but not you all any can had her was one our out has him his how its may new now old see "+ + "two way who did get let say she too use about above after again also been before being below between both "+ + "could does doing down during each few from further have having here into just more most much must only other "+ + "over own same should some such than that their them then there these they this those through under until very "+ + "want what when where which while whom why will with would your yours "+ + "bug bugs issue issues expected behavior behaviour actual reproduce reproduction steps version versions "+ + "current currently describe description problem thanks thank please like think seems seem happen happens "+ + "work works working instead because since using used example following sure able still even though"+ ).split(" "));++ /**+ * How much more a query word counts when it is part of a code name the issue+ * mentions (`extractCandidates`) than when it is only in the prose.+ */+ export const BM25_IDENTIFIER_WEIGHT = 2;++ /** Each query word, with its weight: 1 for prose, more for a word of a named identifier. */-function bm25Query(query: string): Map<string, number> {++export function bm25QueryWeights(query: string): Map<string, number> {+ const weights = new Map<string, number>();+ for (const t of bm25Tokens(query)) if (!BM25_STOPWORDS.has(t)) weights.set(t, 1);+ for (const id of extractCandidates(query).identifiers) {+ for (const t of bm25Tokens(id)) if (weights.has(t)) weights.set(t, BM25_IDENTIFIER_WEIGHT);+ }+ return weights;+ }+export interface Bm25Hit {path: string;score: number;}+/** One file as BM25 reads it: how often each counted word occurs, and its length in words. */+export interface Bm25Doc {+ tf: Map<string, number>;+ length: number;+}++/**+ * A file's text as BM25 reads it, or undefined when it is not scored (too+ * large, or unreadable). `wanted` limits the words counted -- see `bm25Rank`;+ * without it every word is, which is what the on-disk index stores.+ */+export function bm25Doc(path: string, text: string | undefined, wanted?: ReadonlySet<string>): Bm25Doc | undefined {+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES) return undefined;+ const tf = new Map<string, number>();- const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);- for (const t of tokens) if (!wanted || wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);- return { tf, length: tokens.length };++ let length = 0;++ // Counted as the words stream past: no array per file.++ const count = (t: string) => {++ length++;++ if (!wanted || wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);++ };++ eachBm25Token(path.replace(/\//g, " "), count);++ eachBm25Token(text, count);++ return { tf, length };+}+- /** The query's distinct BM25 words, in the order it first uses them. */++/** The query's scored words (stopwords dropped), in the order it first uses them. */+export function bm25QueryTerms(query: string): string[] {- return [...new Set(bm25Tokens(query))];++ return [...bm25QueryWeights(query).keys()];+}+/*** Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files* that share no word with it left out. A file's path is part of its text, so@@@ -305,27 -361,22 +391,27 @@@ export function bm25Rank// Only the query's words are counted: BM25 needs no other term frequency,// and a map of every word of every file is most of the memory and a third// of the time on a large repository. A document's length is still every word.- const terms = bm25QueryTerms(query);- const wanted = new Set(terms);- const weights = bm25Query(query);- const terms = [...weights.keys()];- const docs = new Map<string, { tf: Map<string, number>; length: number }>();++ const weights = bm25QueryWeights(query);++ const wanted = new Set(weights.keys());+ const docs = new Map<string, Bm25Doc>();for (const path of files) {- const text = repo.read(path);- if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES) continue;- const tf = new Map<string, number>();- let length = 0;- const count = (t: string) => {- length++;- if (weights.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);- };- eachBm25Token(path.replace(/\//g, " "), count);- eachBm25Token(text, count);- docs.set(path, { tf, length });+ const doc = bm25Doc(path, repo.read(path), wanted);+ if (doc) docs.set(path, doc);}- return bm25Score(docs, terms, k1, b);++ return bm25Score(docs, weights, k1, b);+}++/**+ * BM25 of already-tokenized files. Split out of `bm25Rank` so the on-disk+ * index (`bm25-cache.ts`) scores through the very same arithmetic: the same+ * documents in, the same scores out, to the last bit.+ */+export function bm25Score(+ docs: ReadonlyMap<string, Bm25Doc>,- terms: string[],++ weights: ReadonlyMap<string, number>,+ k1 = 1.2,+ b = 0.75,+): Bm25Hit[] {if (docs.size === 0) return [];const n = docs.size;let total = 0;@@@ -338,12 -389,12 +424,12 @@@const out: Bm25Hit[] = [];for (const [path, doc] of docs) {let score = 0;-- for (const t of terms) {++ for (const [t, weight] of weights) {const f = doc.tf.get(t);if (!f) continue;const d = df.get(t)!;const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));- score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));- score += weights.get(t)! * idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * doc.length) / avg)));++ score += weight * idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * doc.length) / avg)));}if (score > 0) out.push({ path, score });}diff --cc ix-cli/src/client/api.tsindex fb85eb10,c19077a7..c1b54b14--- a/ix-cli/src/client/api.ts+++ b/ix-cli/src/client/api.ts@@@ -691,51 -662,32 +703,51 @@@ export class IxClient}private async post<T>(path: string, body: unknown): Promise<T> {- const resp = await fetch(`${this.endpoint}${path}`, {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify(body),- // These small reads/writes (source-hashes, stitch, list, ...) had no- // timeout, so a stalled connection could hang the process indefinitely.- // 2 min per request, also bounded by the shared deadline when set.- signal: this.signalFor(2 * 60 * 1000),- });- if (!resp.ok) {- const text = await resp.text();- throw new Error(`${resp.status}: ${text}`);- }- return parseOrThrowWithStatus<T>(resp);+ const payload = JSON.stringify(body);+ return this.read<T>("POST", path, payload, () =>+ fetch(`${this.endpoint}${path}`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: payload,+ // These small reads/writes (source-hashes, stitch, list, ...) had no+ // timeout, so a stalled connection could hang the process indefinitely.+ // 2 min per request, also bounded by the shared deadline when set.+ signal: this.signalFor(2 * 60 * 1000),+ }));}- private async get<T>(path: string): Promise<T> {+ private async get<T>(path: string, timeoutMs = 2 * 60 * 1000): Promise<T> {- const resp = await fetch(`${this.endpoint}${path}`, {- signal: this.signalFor(timeoutMs),- });- if (!resp.ok) {- const text = await resp.text();- throw new Error(`${resp.status}: ${text}`);- }- return parseOrThrowWithStatus<T>(resp);+ return this.read<T>("GET", path, "", () =>+ fetch(`${this.endpoint}${path}`, {- signal: this.signalFor(2 * 60 * 1000),++ signal: this.signalFor(timeoutMs),+ }));}++ /**+ * One request whose body is JSON: sent under the in-flight limit when there+ * is one, shared with an identical earlier request when `shareReads` is on+ * and the path is a read, and parsed per caller either way.+ */+ private async read<T>(method: string, path: string, payload: string, send: () => Promise<Response>): Promise<T> {+ const fetchBody = async (): Promise<SuccessBody> => {+ const run = async (): Promise<SuccessBody> => {+ const resp = await send();+ const text = await resp.text();+ if (!resp.ok) throw new Error(`${resp.status}: ${text}`);+ return { status: resp.status, text };+ };+ return this.limiter ? this.limiter.run(run) : run();+ };+ const body = this.memo && isSharedRead(path)+ ? await this.memo.run(`${method} ${path} ${payload}`, fetchBody)+ : await fetchBody();+ return parseOrThrowWithStatus<T>(body);+ }+}++interface SuccessBody {+ status: number;+ text: string;}/**