Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
// Copyright 2026 Ix Infrastructure Inc.
// issue.ts of fix/issue-start-picking (b62b524), the branch this one is stacked on, frozen as a
// second baseline: its wider isSourcePath and name order, the old BM25. Transpiled; imports repointed.
import { readFileSync } from "node:fs";
import { posix } from "node:path";
import { isTestPath } from "../../dist/cli/explain/related-files.js";
import { resolveToken } from "../../dist/cli/explain/text-references.js";
/**
* From an issue's text to the files its fix starts in, for
* `ix context --from-issue`.
*
* Measured on SWE-PolyBench (28 fresh instances, not used while building
* this): resolving the code names an issue mentions to their definitions put
* a changed file in the top five 0.600 of the time, against 0.485 for BM25
* over the repository. But a graph-expanded bundle, ranked by graph distance,
* lost to BM25 at equal size (0.57 against 0.67). So the ranking here is
* lexical first: the resolved starting files lead, BM25 against the issue
* orders the rest, and graph closeness to a starting point only breaks ties
* and nudges.
*
* Starting points, most specific first:
*
* 1. file paths the issue names, resolved against the tracked files;
* 2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`
* -- from backticks or running text. A multi-part name is almost always a
* code name;
* 3. single backticked words of four letters or more. `save` was the right
* start for a keras fix; `flex`, `code` and `version` were not, which is
* why a single word counts only in backticks, and only after the rest.
*
* A name resolves only to an exact-name definition in source code. The first
* pilot run started from a CSS output file, a changelog, test fixtures, and an
* import: `borderStylesReset` resolved to the file that imported it, because
* the graph's `module` entity for an import is located in the importing file.
*/
/** Starting points a bundle is built from. */
export const MAX_STARTS = 3;
/** Candidate names looked up, in order. */
export const MAX_CANDIDATES = 25;
/** Files larger than this are not scored: generated, bundled or data. */
export const MAX_BM25_BYTES = 200 * 1024;
/** Ranked files carried in the JSON bundle. */
export const MAX_RANKED_FILES = 20;
/**
* How much being next to a starting point is worth, as a fraction of a file's
* BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at
* equal size, so closeness may reorder files the text scores alike and may
* not overturn a clear lexical lead.
*/
export const CLOSENESS_BOOST = 0.1;
/** Extensions of files a fix edits. */
const CODE_EXTENSIONS = [
"ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",
"kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",
];
const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);
/**
* Not code a fix edits: tests, test data, samples, docs, vendored or built
* output. The pilot's list, plus what `isTestPath` already knows.
*
* The second line is a project's website, playground, test configuration,
* benchmarks, stories and end-to-end suites. Svelte's `site/` alone was 40
* of the wrong files in Ix's top-10 rankings over two samples of issues; none
* of these directories holds any of the 932 files the 382 SWE-PolyBench
* Verified fixes change.
*/
const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"
+ "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased|"
+ "site|website|docs-site|playground|tests?[_-]config|benchmarks|\\.?storybook|e2e|cypress|integration-tests)/"
+ "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");
/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */
export function isSourcePath(path) {
return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);
}
// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,
// and the absolute path of a stack trace frame or the tail of a repository URL,
// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which
// `resolveIssuePath` cuts down to the tracked path they end in.
//
// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then
// only begin where a run of path characters begins, so a long unbroken run --
// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.
const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");
const BACKTICK = /`([^`\n]{2,80})`/g;
const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;
// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.
const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";
const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");
const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);
/** The paths and code names an issue mentions, most specific first. */
export function extractCandidates(text) {
const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));
// A path's own parts are not names in the issue: `saving_api` in
// `saving/saving_api.py` is the path again, not a function.
const prose = text.replace(PATH, " ");
const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));
const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));
const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);
// Only MAX_STARTS names become starts, so order decides: a short common
// word (`repeat`, `ignore`) mentioned first took a slot a specific one
// (`Stylesheet`) mentioned later needed. Common words still resolve, last.
const ordered = [
...multi,
...single.filter((t) => !isGenericIdentifier(t)),
...single.filter((t) => isGenericIdentifier(t)),
];
const seen = new Set(paths);
const identifiers = [];
for (const token of ordered) {
if (seen.has(token))
continue;
seen.add(token);
identifiers.push(token);
}
return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };
}
/** Kinds a definition has, most navigable first. Anything else ranks after. */
const DEFINITION_KINDS = [
"class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",
"constant", "variable", "property", "field",
];
/** Never a definition: an import (located in the importer), a chunk, a file. */
const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);
function kindRank(kind) {
const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());
return i === -1 ? DEFINITION_KINDS.length : i;
}
/**
* Up to `max` starting points, first found wins, one per file; and the
* candidates that resolved to nothing. Candidates after the last start are
* not looked up, and not reported as unresolved.
*/
export async function pickStartingPoints(text, deps, max = MAX_STARTS) {
const { paths, identifiers } = extractCandidates(text);
const starts = [];
const unresolved = [];
const taken = (path) => starts.some((s) => s.path === path);
const tracked = new Set(deps.files);
const byBasename = new Map();
for (const file of deps.files) {
const base = posix.basename(file);
byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
}
for (const token of paths) {
if (starts.length >= max)
return { starts, unresolved };
const path = resolveIssuePath(token, tracked, byBasename);
if (!path || !isSourcePath(path)) {
unresolved.push(token);
continue;
}
if (!taken(path))
starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });
}
const score = deps.fileScore ?? (() => 0);
for (const token of identifiers) {
if (starts.length >= max)
break;
// Not caught: a backend that cannot answer is an error to report, not an
// issue that names nothing.
const hits = (await deps.search(token))
.filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))
.sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));
if (hits.length === 0) {
unresolved.push(token);
continue;
}
const best = hits[0];
if (taken(best.path))
continue;
starts.push({
token, id: best.id, name: best.name, kind: best.kind, path: best.path,
...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),
...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),
via: "identifier in issue",
});
}
return { starts, unresolved };
}
/**
* A path as an issue writes it, to the tracked file it names. First as
* `resolveToken` reads a path in code; then, for an absolute path from a
* stack trace or the tail of a repository URL, the longest of its suffixes
* that is a tracked path: `/home/u/proj/src/a.py` and
* `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.
*/
function resolveIssuePath(token, tracked, byBasename) {
const direct = resolveToken(token, "", tracked, byBasename);
if (direct)
return direct;
const parts = token.split("/");
for (let i = 1; i < parts.length; i++) {
const suffix = parts.slice(i).join("/");
if (tracked.has(suffix))
return suffix;
}
return undefined;
}
/**
* Words for BM25: identifiers split at camelCase and underscores, lowercased,
* parts of two letters or fewer dropped. `listByKind` is `list kind`, so an
* issue that says "list by kind" matches the code that spells it as one word.
*/
export function bm25Tokens(text) {
const out = [];
for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {
const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];
for (const part of parts)
if (part.length > 2)
out.push(part.toLowerCase());
}
return out;
}
/**
* Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files
* that share no word with it left out. A file's path is part of its text, so
* `auth/login.ts` matches an issue about logging in before it is opened.
*/
export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {
// Only the query's words are counted: BM25 needs no other term frequency,
// and a map of every word of every file is most of the memory and a third
// of the time on a large repository. A document's length is still every word.
const terms = [...new Set(bm25Tokens(query))];
const wanted = new Set(terms);
const docs = new Map();
for (const path of files) {
const text = repo.read(path);
if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)
continue;
const tf = new Map();
const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);
for (const t of tokens)
if (wanted.has(t))
tf.set(t, (tf.get(t) ?? 0) + 1);
docs.set(path, { tf, length: tokens.length });
}
if (docs.size === 0)
return [];
const n = docs.size;
let total = 0;
const df = new Map();
for (const doc of docs.values()) {
total += doc.length;
for (const t of doc.tf.keys())
df.set(t, (df.get(t) ?? 0) + 1);
}
const avg = total / n || 1;
const out = [];
for (const [path, doc] of docs) {
let score = 0;
for (const t of terms) {
const f = doc.tf.get(t);
if (!f)
continue;
const d = df.get(t);
const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));
score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));
}
if (score > 0)
out.push({ path, score });
}
return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));
}
/**
* Starting points and the lexical ranking for one issue. Without a git
* checkout there is no file list: paths do not resolve, BM25 is empty, and
* only names can start the bundle.
*/
export async function planIssue(text, deps) {
const files = deps.repo?.files() ?? [];
const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];
const scores = new Map(bm25.map((h) => [h.path, h.score]));
const { starts, unresolved } = await pickStartingPoints(text, {
files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,
});
if (starts.length > 0 || bm25.length === 0)
return { starts, unresolved, fallback: false, bm25 };
const top = bm25[0].path;
return {
starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],
unresolved,
fallback: true,
bm25,
};
}
/**
* Whether an identifier the issue names is too common to point anywhere: a
* short all-lowercase word such as `debug`, `bind`, `config` or `render`
* resolves to *a* definition, rarely the one the fix touches. Specific names
* carry an inner capital (`toJsonTree`, `JsonWriter`), an underscore
* (`get_openai_callback`), or length.
*
* Fixed before it was scored on held-out samples: on SWE-PolyBench issues the
* starts it keeps were in the gold patch 20 times in 32, the ones it drops 2
* in 7.
*/
export function isGenericIdentifier(token) {
return !(/[a-z][A-Z]/.test(token) || /^[A-Z][a-z]+[A-Z]/.test(token) || token.includes("_") || token.length >= 8);
}
/**
* A plan is confident when the issue named a path, or a specific identifier,
* that resolved. A BM25 fallback never is: across four samples of
* SWE-PolyBench issues its start was in the gold patch once in 55.
*/
export function issueConfidence(plan) {
if (plan.fallback) {
return { confident: false, starts: [], reason: "no code name in the issue resolved to a definition" };
}
const starts = plan.starts.filter((s) => s.via === "path in issue"
|| (s.via === "identifier in issue" && !isGenericIdentifier(s.token)));
if (starts.length === 0) {
const names = plan.starts.map((s) => s.token);
return {
confident: false,
starts,
reason: names.length > 0
? `only common words resolved (${names.join(", ")}), which name many definitions`
: "nothing in the issue resolved to a definition",
};
}
return { confident: true, starts };
}
/** Files after the starting points in a lean view. */
export const LEAN_RANKED = 4;
/**
* The lean view of a plan. A full bundle costs about 1.5k tokens, and an
* agent re-reads it on every turn; on SWE-PolyBench it added a median 14%
* tokens even where its start was right, and saved no turns. What the agent
* uses is the pointer, so that is all this keeps: the trusted starting points
* and a few ranked files -- Ix's starts, then BM25, the ordering that beat
* both alone on held-out issues. When nothing is trusted it says so, rather
* than pointing the agent at a guess.
*/
export function leanIssueView(plan, ranked = LEAN_RANKED) {
const c = issueConfidence(plan);
if (!c.confident) {
return { confidence: "low", reason: c.reason, startingPoints: [], alsoRanked: [], unresolved: plan.unresolved };
}
const startPaths = new Set(c.starts.map((s) => s.path));
const alsoRanked = rankIssueFiles({ starts: [], bm25: plan.bm25, near: new Map() }, ranked + startPaths.size)
.filter((f) => !startPaths.has(f.path))
.slice(0, ranked);
return { confidence: "high", startingPoints: c.starts, alsoRanked, unresolved: plan.unresolved };
}
/** BM25 files tried, best first, for one the graph has a node for. */
export const CENTRE_FALLBACK_TRIES = 10;
/**
* The starting point a bundle is centred on: the first start the graph has a
* node for, else the best BM25 file that has one.
*
* A start can have no node: the graph does not index every tracked file.
* On SWE-PolyBench two svelte issues named no code, BM25's top file was a
* `.svelte` component, and the command failed with no bundle at all while
* the fifth-ranked file was a perfectly good `.js` module. Walking BM25's
* ranking finds that one; the files the walk skipped still rank as usual.
* `walked` says the centre came from this walk, which is a fallback too.
*/
export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {
const located = await Promise.all(starts.map(async (s) => {
const id = s.id ?? (await findNode(s.path));
return id ? { ...s, id } : s;
}));
const found = located.find((s) => s.id);
if (found)
return { starts: located, centre: found, walked: false };
const tried = new Set(located.map((s) => s.path));
for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {
const id = await findNode(hit.path);
if (!id)
continue;
const centre = {
token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",
};
return { starts: [...located, centre], centre, walked: true };
}
return { starts: located, walked: false };
}
/**
* The files to read, best first: the starting files in the order they were
* found, then every other source file by
*
* bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))
*
* where closeness is 1 for a file one hop from a starting point, 0.5 for one
* of its ranked related files, 0 otherwise. Ties go to the closer file, then
* the path. A graph neighbour the issue's text does not match at all still
* ranks, after every file it does match.
*/
export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {
const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));
const out = [];
const startPaths = new Set();
for (const s of input.starts) {
if (startPaths.has(s.path))
continue;
startPaths.add(s.path);
out.push({
path: s.path,
score: round(lexical.get(s.path) ?? 0),
reason: `starting point (${s.via}: ${s.token})`,
});
}
const pool = new Set([...lexical.keys(), ...input.near.keys()]);
const rest = [...pool]
.filter((path) => !startPaths.has(path) && isSourcePath(path))
.map((path) => {
const bm25 = lexical.get(path) ?? 0;
const near = input.near.get(path);
const closeness = near?.weight ?? 0;
const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]
.filter(Boolean).join("; ");
return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };
})
.sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));
for (const r of rest)
out.push({ path: r.path, score: round(r.score), reason: r.reason });
return out.slice(0, limit);
}
/**
* The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the
* stream can be replaced under test.
*/
export async function readIssueText(arg, stdin = process.stdin) {
if (arg !== "-")
return decodeIssue(readFileSync(arg));
const chunks = [];
for await (const chunk of stdin) {
chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));
}
return decodeIssue(Buffer.concat(chunks));
}
/**
* UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`
* writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >
* issue.md` there produces a file that, read as UTF-8, is every letter
* followed by a NUL: no name, no path, and no BM25 word in it.
*/
function decodeIssue(bytes) {
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
return bytes.subarray(2).toString("utf16le");
}
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));
return swapped.swap16().toString("utf16le");
}
const text = bytes.toString("utf8");
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
}
function unique(items) {
return [...new Set(items)];
}
function round(n) {
return Math.round(n * 1000) / 1000;
}
function cmp(a, b) {
return a < b ? -1 : a > b ? 1 : 0;
}