Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

ranking-eval: the offline BM25 study for --from-issue, summaries only

KageBinary023735917e1e
diff --git a/.gitignore b/.gitignore
index 6e82a0d4..46dcf306 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,4 +46,4 @@ ix-cli/.watch-child-runtime-*
!.env.example
# SBT boot artifacts
-.sbt-boot/
\ No newline at end of file
+.sbt-boot/ix-cli/scripts/ranking-eval/results/*.jsonl
diff --git a/ix-cli/scripts/ranking-eval/baseline-issue.mjs b/ix-cli/scripts/ranking-eval/baseline-issue.mjs
new file mode 100644
index 00000000..3d3228c6
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/baseline-issue.mjs
@@ -0,0 +1,383 @@
+// Copyright 2026 Ix Infrastructure Inc.
+// The compiled issue.ts of origin/main 198cebd (unchanged at abf39c8), before this branch,
+// frozen so the baseline can be scored after issue.ts changes. tsc output, imports repointed.
+import { readFileSync } from "node:fs";
+import { posix } from "node:path";
+import { isTestPath } from "../../dist/cli/explain/related-files.js";
+import { resolveToken } from "../../dist/cli/explain/text-references.js";
+/**
+ * From an issue's text to the files its fix starts in, for
+ * `ix context --from-issue`.
+ *
+ * Measured on SWE-PolyBench (28 fresh instances, not used while building
+ * this): resolving the code names an issue mentions to their definitions put
+ * a changed file in the top five 0.600 of the time, against 0.485 for BM25
+ * over the repository. But a graph-expanded bundle, ranked by graph distance,
+ * lost to BM25 at equal size (0.57 against 0.67). So the ranking here is
+ * lexical first: the resolved starting files lead, BM25 against the issue
+ * orders the rest, and graph closeness to a starting point only breaks ties
+ * and nudges.
+ *
+ * Starting points, most specific first:
+ *
+ * 1. file paths the issue names, resolved against the tracked files;
+ * 2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`
+ * -- from backticks or running text. A multi-part name is almost always a
+ * code name;
+ * 3. single backticked words of four letters or more. `save` was the right
+ * start for a keras fix; `flex`, `code` and `version` were not, which is
+ * why a single word counts only in backticks, and only after the rest.
+ *
+ * A name resolves only to an exact-name definition in source code. The first
+ * pilot run started from a CSS output file, a changelog, test fixtures, and an
+ * import: `borderStylesReset` resolved to the file that imported it, because
+ * the graph's `module` entity for an import is located in the importing file.
+ */
+/** Starting points a bundle is built from. */
+export const MAX_STARTS = 3;
+/** Candidate names looked up, in order. */
+export const MAX_CANDIDATES = 25;
+/** Files larger than this are not scored: generated, bundled or data. */
+export const MAX_BM25_BYTES = 200 * 1024;
+/** Ranked files carried in the JSON bundle. */
+export const MAX_RANKED_FILES = 20;
+/**
+ * How much being next to a starting point is worth, as a fraction of a file's
+ * BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at
+ * equal size, so closeness may reorder files the text scores alike and may
+ * not overturn a clear lexical lead.
+ */
+export const CLOSENESS_BOOST = 0.1;
+/** Extensions of files a fix edits. */
+const CODE_EXTENSIONS = [
+ "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",
+ "kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",
+];
+const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);
+/**
+ * Not code a fix edits: tests, test data, samples, docs, vendored or built
+ * output. The pilot's list, plus what `isTestPath` already knows.
+ */
+const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"
+ + "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased)/"
+ + "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");
+/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */
+export function isSourcePath(path) {
+ return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);
+}
+// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,
+// and the absolute path of a stack trace frame or the tail of a repository URL,
+// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which
+// `resolveIssuePath` cuts down to the tracked path they end in.
+//
+// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then
+// only begin where a run of path characters begins, so a long unbroken run --
+// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.
+const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");
+const BACKTICK = /`([^`\n]{2,80})`/g;
+const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;
+// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.
+const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";
+const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");
+const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);
+/** The paths and code names an issue mentions, most specific first. */
+export function extractCandidates(text) {
+ const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));
+ // A path's own parts are not names in the issue: `saving_api` in
+ // `saving/saving_api.py` is the path again, not a function.
+ const prose = text.replace(PATH, " ");
+ const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));
+ const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));
+ const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);
+ const seen = new Set(paths);
+ const identifiers = [];
+ for (const token of [...multi, ...single]) {
+ if (seen.has(token))
+ continue;
+ seen.add(token);
+ identifiers.push(token);
+ }
+ return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };
+}
+/** Kinds a definition has, most navigable first. Anything else ranks after. */
+const DEFINITION_KINDS = [
+ "class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",
+ "constant", "variable", "property", "field",
+];
+/** Never a definition: an import (located in the importer), a chunk, a file. */
+const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);
+function kindRank(kind) {
+ const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());
+ return i === -1 ? DEFINITION_KINDS.length : i;
+}
+/**
+ * Up to `max` starting points, first found wins, one per file; and the
+ * candidates that resolved to nothing. Candidates after the last start are
+ * not looked up, and not reported as unresolved.
+ */
+export async function pickStartingPoints(text, deps, max = MAX_STARTS) {
+ const { paths, identifiers } = extractCandidates(text);
+ const starts = [];
+ const unresolved = [];
+ const taken = (path) => starts.some((s) => s.path === path);
+ const tracked = new Set(deps.files);
+ const byBasename = new Map();
+ for (const file of deps.files) {
+ const base = posix.basename(file);
+ byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
+ }
+ for (const token of paths) {
+ if (starts.length >= max)
+ return { starts, unresolved };
+ const path = resolveIssuePath(token, tracked, byBasename);
+ if (!path || !isSourcePath(path)) {
+ unresolved.push(token);
+ continue;
+ }
+ if (!taken(path))
+ starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });
+ }
+ const score = deps.fileScore ?? (() => 0);
+ for (const token of identifiers) {
+ if (starts.length >= max)
+ break;
+ // Not caught: a backend that cannot answer is an error to report, not an
+ // issue that names nothing.
+ const hits = (await deps.search(token))
+ .filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))
+ .sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));
+ if (hits.length === 0) {
+ unresolved.push(token);
+ continue;
+ }
+ const best = hits[0];
+ if (taken(best.path))
+ continue;
+ starts.push({
+ token, id: best.id, name: best.name, kind: best.kind, path: best.path,
+ ...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),
+ ...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),
+ via: "identifier in issue",
+ });
+ }
+ return { starts, unresolved };
+}
+/**
+ * A path as an issue writes it, to the tracked file it names. First as
+ * `resolveToken` reads a path in code; then, for an absolute path from a
+ * stack trace or the tail of a repository URL, the longest of its suffixes
+ * that is a tracked path: `/home/u/proj/src/a.py` and
+ * `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.
+ */
+function resolveIssuePath(token, tracked, byBasename) {
+ const direct = resolveToken(token, "", tracked, byBasename);
+ if (direct)
+ return direct;
+ const parts = token.split("/");
+ for (let i = 1; i < parts.length; i++) {
+ const suffix = parts.slice(i).join("/");
+ if (tracked.has(suffix))
+ return suffix;
+ }
+ return undefined;
+}
+/**
+ * Words for BM25: identifiers split at camelCase and underscores, lowercased,
+ * parts of two letters or fewer dropped. `listByKind` is `list kind`, so an
+ * issue that says "list by kind" matches the code that spells it as one word.
+ */
+export function bm25Tokens(text) {
+ const out = [];
+ for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {
+ const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];
+ for (const part of parts)
+ if (part.length > 2)
+ out.push(part.toLowerCase());
+ }
+ return out;
+}
+/**
+ * Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files
+ * that share no word with it left out. A file's path is part of its text, so
+ * `auth/login.ts` matches an issue about logging in before it is opened.
+ */
+export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {
+ // Only the query's words are counted: BM25 needs no other term frequency,
+ // and a map of every word of every file is most of the memory and a third
+ // of the time on a large repository. A document's length is still every word.
+ const terms = [...new Set(bm25Tokens(query))];
+ const wanted = new Set(terms);
+ const docs = new Map();
+ for (const path of files) {
+ const text = repo.read(path);
+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)
+ continue;
+ const tf = new Map();
+ const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);
+ for (const t of tokens)
+ if (wanted.has(t))
+ tf.set(t, (tf.get(t) ?? 0) + 1);
+ docs.set(path, { tf, length: tokens.length });
+ }
+ if (docs.size === 0)
+ return [];
+ const n = docs.size;
+ let total = 0;
+ const df = new Map();
+ for (const doc of docs.values()) {
+ total += doc.length;
+ for (const t of doc.tf.keys())
+ df.set(t, (df.get(t) ?? 0) + 1);
+ }
+ const avg = total / n || 1;
+ const out = [];
+ for (const [path, doc] of docs) {
+ let score = 0;
+ for (const t of terms) {
+ const f = doc.tf.get(t);
+ if (!f)
+ continue;
+ const d = df.get(t);
+ const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));
+ score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));
+ }
+ if (score > 0)
+ out.push({ path, score });
+ }
+ return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));
+}
+/**
+ * Starting points and the lexical ranking for one issue. Without a git
+ * checkout there is no file list: paths do not resolve, BM25 is empty, and
+ * only names can start the bundle.
+ */
+export async function planIssue(text, deps) {
+ const files = deps.repo?.files() ?? [];
+ const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];
+ const scores = new Map(bm25.map((h) => [h.path, h.score]));
+ const { starts, unresolved } = await pickStartingPoints(text, {
+ files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,
+ });
+ if (starts.length > 0 || bm25.length === 0)
+ return { starts, unresolved, fallback: false, bm25 };
+ const top = bm25[0].path;
+ return {
+ starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],
+ unresolved,
+ fallback: true,
+ bm25,
+ };
+}
+/** BM25 files tried, best first, for one the graph has a node for. */
+export const CENTRE_FALLBACK_TRIES = 10;
+/**
+ * The starting point a bundle is centred on: the first start the graph has a
+ * node for, else the best BM25 file that has one.
+ *
+ * A start can have no node: the graph does not index every tracked file.
+ * On SWE-PolyBench two svelte issues named no code, BM25's top file was a
+ * `.svelte` component, and the command failed with no bundle at all while
+ * the fifth-ranked file was a perfectly good `.js` module. Walking BM25's
+ * ranking finds that one; the files the walk skipped still rank as usual.
+ * `walked` says the centre came from this walk, which is a fallback too.
+ */
+export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {
+ const located = await Promise.all(starts.map(async (s) => {
+ const id = s.id ?? (await findNode(s.path));
+ return id ? { ...s, id } : s;
+ }));
+ const found = located.find((s) => s.id);
+ if (found)
+ return { starts: located, centre: found, walked: false };
+ const tried = new Set(located.map((s) => s.path));
+ for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {
+ const id = await findNode(hit.path);
+ if (!id)
+ continue;
+ const centre = {
+ token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",
+ };
+ return { starts: [...located, centre], centre, walked: true };
+ }
+ return { starts: located, walked: false };
+}
+/**
+ * The files to read, best first: the starting files in the order they were
+ * found, then every other source file by
+ *
+ * bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))
+ *
+ * where closeness is 1 for a file one hop from a starting point, 0.5 for one
+ * of its ranked related files, 0 otherwise. Ties go to the closer file, then
+ * the path. A graph neighbour the issue's text does not match at all still
+ * ranks, after every file it does match.
+ */
+export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {
+ const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));
+ const out = [];
+ const startPaths = new Set();
+ for (const s of input.starts) {
+ if (startPaths.has(s.path))
+ continue;
+ startPaths.add(s.path);
+ out.push({
+ path: s.path,
+ score: round(lexical.get(s.path) ?? 0),
+ reason: `starting point (${s.via}: ${s.token})`,
+ });
+ }
+ const pool = new Set([...lexical.keys(), ...input.near.keys()]);
+ const rest = [...pool]
+ .filter((path) => !startPaths.has(path) && isSourcePath(path))
+ .map((path) => {
+ const bm25 = lexical.get(path) ?? 0;
+ const near = input.near.get(path);
+ const closeness = near?.weight ?? 0;
+ const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]
+ .filter(Boolean).join("; ");
+ return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };
+ })
+ .sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));
+ for (const r of rest)
+ out.push({ path: r.path, score: round(r.score), reason: r.reason });
+ return out.slice(0, limit);
+}
+/**
+ * The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the
+ * stream can be replaced under test.
+ */
+export async function readIssueText(arg, stdin = process.stdin) {
+ if (arg !== "-")
+ return decodeIssue(readFileSync(arg));
+ const chunks = [];
+ for await (const chunk of stdin) {
+ chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));
+ }
+ return decodeIssue(Buffer.concat(chunks));
+}
+/**
+ * UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`
+ * writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >
+ * issue.md` there produces a file that, read as UTF-8, is every letter
+ * followed by a NUL: no name, no path, and no BM25 word in it.
+ */
+function decodeIssue(bytes) {
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
+ return bytes.subarray(2).toString("utf16le");
+ }
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+ const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));
+ return swapped.swap16().toString("utf16le");
+ }
+ const text = bytes.toString("utf8");
+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
+function unique(items) {
+ return [...new Set(items)];
+}
+function round(n) {
+ return Math.round(n * 1000) / 1000;
+}
+function cmp(a, b) {
+ return a < b ? -1 : a > b ? 1 : 0;
+}
diff --git a/ix-cli/scripts/ranking-eval/configs.mjs b/ix-cli/scripts/ranking-eval/configs.mjs
new file mode 100644
index 00000000..f73f4a6b
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/configs.mjs
@@ -0,0 +1,99 @@
+// Named variants. Round 1 changes one thing from `base` each; later rounds
+// combine what won on dev.
+const DOCS_SITE = /(^|\/)(site|website|guides|demos?|playground|benchmarks?)\//;
+const DTS = /\.d\.ts$/;
+
+export const VARIANTS = {
+ base: {},
+ // tokenizer
+ code: { code: true },
+ stop: { stop: true },
+ min4: { minLen: 4 },
+ // query weighting
+ qtf: { k3: 1.5 },
+ qtf8: { k3: 8 },
+ id2: { idW: 2 },
+ id3: { idW: 3 },
+ id5: { idW: 5 },
+ id3p: { idW: 3, idPaths: true },
+ // BM25F path field
+ path1: { pathW: 1 },
+ path2: { pathW: 2 },
+ path4: { pathW: 4 },
+ path2base: { pathW: 2, pathField: "base" },
+ path4base: { pathW: 4, pathField: "base" },
+ // length normalisation
+ k1_09: { k1: 0.9 },
+ k1_15: { k1: 1.5 },
+ k1_20: { k1: 2.0 },
+ b03: { b: 0.3 },
+ b05: { b: 0.5 },
+ b09: { b: 0.9 },
+ cap3: { lenCap: 3 },
+ max100: { maxBytes: 100 * 1024 },
+ // file kinds
+ nosite: { penalty: (p) => (DOCS_SITE.test(p) ? 0.5 : 1) },
+ nodts: { penalty: (p) => (DTS.test(p) ? 0.5 : 1) },
+};
+
+const CS = { code: true, stop: true };
+Object.assign(VARIANTS, {
+ cs: CS,
+ cs_q8: { ...CS, k3: 8 },
+ cs_id2: { ...CS, idW: 2 },
+ cs_id3: { ...CS, idW: 3 },
+ cs_q8_id2: { ...CS, k3: 8, idW: 2 },
+ cs_site: { ...CS, penalty: VARIANTS.nosite.penalty },
+ cs_k15: { ...CS, k1: 1.5 },
+ cs_k20: { ...CS, k1: 2.0 },
+ cs_big05: { ...CS, bigram: 0.5 },
+ cs_big1: { ...CS, bigram: 1 },
+ cs_path2base: { ...CS, pathW: 2, pathField: "base" },
+ max256: { maxBytes: 256 * 1024 },
+ code_big1: { code: true, bigram: 1 },
+});
+
+// Round 3: on top of code tokens + stopwords + doc-site penalty.
+const CSS = { ...CS, penalty: VARIANTS.nosite.penalty };
+const only = (re) => (p) => (re.test(p) ? 0 : 1);
+Object.assign(VARIANTS, {
+ cs_site0: { ...CS, penalty: only(DOCS_SITE) },
+ cs_siteonly: { ...CS, penalty: only(/(^|\/)(site|website)\//) },
+ css_id2: { ...CSS, idW: 2 },
+ css_id3: { ...CSS, idW: 3 },
+ css_q8: { ...CSS, k3: 8 },
+ css_q2: { ...CSS, k3: 2 },
+ css_k15: { ...CSS, k1: 1.5 },
+ css_k10: { ...CSS, k1: 1.0 },
+ css_b06: { ...CSS, b: 0.6 },
+ css_b09: { ...CSS, b: 0.9 },
+ css_big05: { ...CSS, bigram: 0.5 },
+});
+
+// The chosen variant, as issue.ts implements it: code tokens, prose
+// stopwords, identifier terms x2, and docs-site directories out of the corpus.
+Object.assign(VARIANTS, {
+ final: { ...CS, idW: 2, exclude: DOCS_SITE },
+});
+
+// Ablations of `final`: each leaves one of its four parts out.
+Object.assign(VARIANTS, {
+ final_nocode: { stop: true, idW: 2, exclude: DOCS_SITE },
+ final_nostop: { code: true, idW: 2, exclude: DOCS_SITE },
+ final_noid: { ...CS, exclude: DOCS_SITE },
+ final_nosite: { ...CS, idW: 2 },
+});
+
+// v2, after the coordinator's check against all 382 issues' gold: guides/,
+// demo(s)/ and benchmark/ hold files real fixes change, so an exclusion rule
+// must hold across the whole dataset's gold, not just dev's. isSourcePath now
+// comes from fix/issue-start-picking, whose list was checked against all 932
+// gold files; `final2` is `final` with that list in place of DOCS_SITE.
+const PICKS_NOISE = /(^|\/)(site|website|docs-site|playground|tests?[_-]config|benchmarks|\.?storybook|e2e|cypress|integration-tests)\//;
+Object.assign(VARIANTS, {
+ picks_base: { exclude: PICKS_NOISE },
+ final2: { ...CS, idW: 2, exclude: PICKS_NOISE },
+ final2_nocode: { stop: true, idW: 2, exclude: PICKS_NOISE },
+ final2_nostop: { code: true, idW: 2, exclude: PICKS_NOISE },
+ final2_noid: { ...CS, exclude: PICKS_NOISE },
+});
diff --git a/ix-cli/scripts/ranking-eval/corpus.mjs b/ix-cli/scripts/ranking-eval/corpus.mjs
new file mode 100644
index 00000000..2f0e4664
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/corpus.mjs
@@ -0,0 +1,58 @@
+// Read a repository snapshot straight from ix-bench's bare clones: no
+// checkout, no backend. `git ls-tree` lists the tracked files at the base
+// commit; `git cat-file --batch` reads the source ones in one process.
+import { execFileSync, spawnSync } from "node:child_process";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { isSourcePath } from "./baseline-issue.mjs";
+
+export const REPOS = join(homedir(), ".cache", "ix-bench", "external", "repos");
+/** What `gitRepoAccess` reads at most (text-references MAX_SCAN_BYTES). */
+const MAX_READ_BYTES = 256 * 1024;
+
+export function bareRepo(repo) {
+ return join(REPOS, `${repo.replace("/", "__")}.git`);
+}
+
+/** A RepoAccess over `commit`: every tracked file listed, source files readable. */
+export function loadCorpus(repo, commit) {
+ const cwd = bareRepo(repo);
+ const listing = execFileSync("git", ["ls-tree", "-r", "-z", "--long", commit], {
+ cwd, encoding: "utf8", maxBuffer: 256 * 1024 * 1024,
+ }).split("\0").filter(Boolean);
+ const files = [];
+ const wanted = [];
+ for (const row of listing) {
+ const tab = row.indexOf("\t");
+ const [, type, oid, size] = row.slice(0, tab).trim().split(/\s+/);
+ const path = row.slice(tab + 1);
+ if (type !== "blob") continue;
+ files.push(path);
+ if (isSourcePath(path) && Number(size) <= MAX_READ_BYTES) wanted.push({ path, oid });
+ }
+ const texts = new Map();
+ if (wanted.length) {
+ const out = spawnSync("git", ["cat-file", "--batch"], {
+ cwd, input: wanted.map((w) => w.oid).join("\n") + "\n", maxBuffer: 2 * 1024 * 1024 * 1024,
+ }).stdout;
+ let pos = 0;
+ for (const w of wanted) {
+ const nl = out.indexOf(10, pos);
+ const header = out.toString("utf8", pos, nl).split(" ");
+ const len = Number(header[2]);
+ texts.set(w.path, out.toString("utf8", nl + 1, nl + 1 + len));
+ pos = nl + 1 + len + 1;
+ }
+ }
+ return {
+ files: () => files,
+ read: (path) => texts.get(path),
+ grep: () => [],
+ texts,
+ };
+}
+
+export function goldFiles(patch) {
+ return [...new Set([...patch.matchAll(/^diff --git a\/(\S+) b\//gm)].map((m) => m[1]))];
+}
diff --git a/ix-cli/scripts/ranking-eval/defs.mjs b/ix-cli/scripts/ranking-eval/defs.mjs
new file mode 100644
index 00000000..0e2592af
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/defs.mjs
@@ -0,0 +1,37 @@
+// A stand-in for `ix search --name-only` with no backend: the definitions of
+// a name, found by regex over the snapshot's source files. Only good enough to
+// measure the plan's ranking (starting files, then BM25) offline; the graph's
+// own resolver is what the CLI uses.
+const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+const LEADING_NOT_DEF = /\b(return|new|if|while|for|switch|catch|else|await|throw|yield|typeof|case)\b/;
+
+export function defSearch(repo, files) {
+ return async (name) => {
+ const n = esc(name);
+ const pats = [
+ [new RegExp(`\\b(class|interface|enum|struct|trait|record|object)\\s+${n}\\b`), (m) => (m[1] === "record" ? "class" : m[1])],
+ [new RegExp(`\\btype\\s+${n}\\b\\s*[=<]`), () => "type"],
+ [new RegExp(`\\b(?:function\\s*\\*?|def|fun|fn|func)\\s+${n}\\b`), () => "function"],
+ [new RegExp(`\\b(?:const|let|var)\\s+${n}\\s*=\\s*(?:async\\s*)?(?:function\\b|\\([^)]*\\)\\s*=>|[A-Za-z_$][\\w$]*\\s*=>)`), () => "function"],
+ [new RegExp(`^[ \\t]*((?:[\\w<>\\[\\],.?@]+[ \\t]+)*)${n}[ \\t]*(?:<[^>\\n]*>)?\\([^\\n;]*\\)[^\\n;]*\\{[ \\t]*$`, "m"),
+ (m) => (LEADING_NOT_DEF.test(m[1]) ? undefined : "method")],
+ [new RegExp(`\\b(?:const|let|var|val|final\\s+[\\w<>\\[\\]]+|static\\s+[\\w<>\\[\\]]+)\\s+${n}\\b`), () => (/^[A-Z0-9_]+$/.test(name) ? "constant" : "variable")],
+ [new RegExp(`^${n}\\s*=`, "m"), () => (/^[A-Z0-9_]+$/.test(name) ? "constant" : "variable")],
+ ];
+ const hits = [];
+ for (const path of files) {
+ const text = repo.read(path);
+ if (!text || !text.includes(name)) continue;
+ for (const [re, kindOf] of pats) {
+ const m = text.match(re);
+ const kind = m && kindOf(m);
+ if (kind) {
+ hits.push({ id: `${path}#${name}`, name, kind, path });
+ break;
+ }
+ }
+ }
+ return hits;
+ };
+}
diff --git a/ix-cli/scripts/ranking-eval/eval.mjs b/ix-cli/scripts/ranking-eval/eval.mjs
new file mode 100644
index 00000000..c7f1a293
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/eval.mjs
@@ -0,0 +1,98 @@
+// Score BM25 variants on the dev (or, once, the test) split.
+//
+// node eval.mjs --set dev --variants base,code --out results/dev.jsonl [--plan] [--check]
+//
+// For each instance: read the snapshot at base_commit from ix-bench's bare
+// clone, rank the tracked source files against the problem statement with
+// each variant, and record file recall @5/@10/@20 against the gold patch's
+// source files that exist at the base commit. --plan also scores the plan's
+// order (starting files from pickStartingPoints with a regex definition
+// search standing in for the graph, then BM25) -- rankIssueFiles with no
+// graph closeness. --check asserts `base` equals the baseline bm25Rank.
+// The variant `impl` is the built dist/ (the current issue.ts): its own
+// isSourcePath, bm25Rank, pickStartingPoints and rankIssueFiles. The variant
+// `picks` is the same for fix/issue-start-picking, the branch this is on.
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import * as dist from "../../dist/cli/explain/issue.js";
+// Gold, the file list and the `base` plan use issue.ts as it was before this
+// branch, so the baseline and the scoring stay fixed while issue.ts changes.
+import { bm25Rank, isSourcePath, pickStartingPoints, rankIssueFiles } from "./baseline-issue.mjs";
+import * as picks from "./picks-issue.mjs";
+import { goldFiles, loadCorpus } from "./corpus.mjs";
+import { defSearch } from "./defs.mjs";
+import { VARIANTS } from "./configs.mjs";
+import { rank } from "./variants.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
+ if (a.startsWith("--")) acc.push([a.slice(2), all[i + 1] && !all[i + 1].startsWith("--") ? all[i + 1] : true]);
+ return acc;
+}, []));
+const set = args.set ?? "dev";
+if (set === "test" && args.final !== "yes-once") {
+ console.error("The test split is scored once, for the final variant: pass --final yes-once.");
+ process.exit(2);
+}
+const split = JSON.parse(readFileSync(`${HERE}/split.json`, "utf8"));
+const ids = new Set(split[set]);
+const only = args.only ? new Set(String(args.only).split(",")) : undefined;
+const rows = readFileSync("/home/ihock/src/ix-bench/results/external/polybench_verified.jsonl", "utf8")
+ .split("\n").filter(Boolean).map((l) => JSON.parse(l))
+ .filter((r) => ids.has(r.instance_id) && (!only || only.has(r.instance_id)));
+const MODULES = { impl: dist, picks };
+const variantNames = String(args.variants ?? "base").split(",");
+for (const v of variantNames) if (!VARIANTS[v] && !MODULES[v]) throw new Error(`unknown variant ${v}`);
+const out = args.out ?? `${HERE}/results/${set}.jsonl`;
+mkdirSync(dirname(out), { recursive: true });
+
+const recall = (pred, gold) => gold.filter((g) => pred.includes(g)).length / gold.length;
+const at = (list, gold) => [5, 10, 20].map((k) => recall(list.slice(0, k), gold));
+
+const lines = [];
+let done = 0;
+for (const row of rows) {
+ const t0 = Date.now();
+ const repo = loadCorpus(row.repo, row.base_commit);
+ const all = repo.files();
+ const tracked = new Set(all);
+ const source = all.filter(isSourcePath);
+ const gold = goldFiles(row.patch).filter((p) => isSourcePath(p) && tracked.has(p));
+ const loadMs = Date.now() - t0;
+ if (gold.length === 0) {
+ lines.push({ id: row.instance_id, lang: row.language, repo: row.repo, skipped: "no source gold at base" });
+ continue;
+ }
+ const query = row.problem_statement;
+ if (args.check) {
+ const a = bm25Rank(repo, source, query).map((h) => h.path).slice(0, 50);
+ const b = rank(repo, source, query, VARIANTS.base).map((h) => h.path).slice(0, 50);
+ if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error(`base != bm25Rank on ${row.instance_id}`);
+ }
+ const search = args.plan ? defSearch(repo, source) : undefined;
+ for (const v of variantNames) {
+ const t1 = Date.now();
+ const mod = MODULES[v];
+ const hits = mod ? mod.bm25Rank(repo, all.filter(mod.isSourcePath), query) : rank(repo, source, query, VARIANTS[v]);
+ const ms = Date.now() - t1;
+ const ranked = hits.map((h) => h.path);
+ const rec = { id: row.instance_id, lang: row.language, repo: row.repo, variant: v, gold, ms, loadMs,
+ nfiles: source.length, bm25: at(ranked, gold), top: ranked.slice(0, 20) };
+ if (search) {
+ // Starting points depend on BM25 only through fileScore's tie-break.
+ const scores = new Map(hits.map((h) => [h.path, h.score]));
+ const lib = mod ?? { pickStartingPoints, rankIssueFiles };
+ const { starts } = await lib.pickStartingPoints(query, { files: all, search, fileScore: (p) => scores.get(p) ?? 0 });
+ const planned = lib.rankIssueFiles({ starts, bm25: hits, near: new Map() }).map((r) => r.path);
+ rec.plan = at(planned, gold);
+ rec.starts = starts.map((s) => s.path);
+ }
+ lines.push(rec);
+ }
+ done++;
+ process.stderr.write(`${done}/${rows.length} ${row.instance_id} files=${source.length} load=${loadMs}ms\n`);
+}
+writeFileSync(out, lines.map((l) => JSON.stringify(l)).join("\n") + "\n");
+console.error(`wrote ${out}`);
diff --git a/ix-cli/scripts/ranking-eval/make_split.py b/ix-cli/scripts/ranking-eval/make_split.py
new file mode 100644
index 00000000..6c9b97bf
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/make_split.py
@@ -0,0 +1,41 @@
+"""Fix the dev/test split before any results are looked at.
+
+DEV = every instance id in the earlier ix-bench runs (tuning allowed).
+TEST = 60 instances from the rest, seed 20260930, stratified by language
+ (15 per language), scored once at the end.
+"""
+import json, random, collections, pathlib
+EXT = pathlib.Path("/home/ihock/src/ix-bench/results/external")
+DEV_FILES = ["polybench_pilot.jsonl", "polybench_v2_first.jsonl", "polybench_v2_heldout.jsonl",
+ "polybench_v3_first.jsonl", "polybench_v3_heldout.jsonl", "polybench_v3_fresh.jsonl",
+ "from_issue_accept.jsonl"]
+dev = set()
+for f in DEV_FILES:
+ for line in open(EXT / f):
+ if line.strip():
+ dev.add(json.loads(line)["instance_id"])
+dev |= set(json.load(open(EXT / "e2e_instances.json")))
+rows = [json.loads(l) for l in open(EXT / "polybench_verified.jsonl")]
+ids = {r["instance_id"] for r in rows}
+dev &= ids
+rest = sorted((r for r in rows if r["instance_id"] not in dev), key=lambda r: r["instance_id"])
+by_lang = collections.defaultdict(list)
+for r in rest:
+ by_lang[r["language"]].append(r["instance_id"])
+rng = random.Random(20260930)
+test = []
+langs = sorted(by_lang)
+per = 60 // len(langs)
+for lang in langs:
+ pool = by_lang[lang]
+ test += rng.sample(pool, min(per, len(pool)))
+# top up if a language was short
+short = 60 - len(test)
+if short:
+ left = [i for l in langs for i in by_lang[l] if i not in test]
+ test += rng.sample(left, short)
+json.dump({"seed": 20260930, "dev": sorted(dev), "test": sorted(test)}, open("split.json", "w"), indent=1)
+lang = {r["instance_id"]: r["language"] for r in rows}
+print("dev", len(dev), collections.Counter(lang[i] for i in dev))
+print("rest", len(rest), {l: len(v) for l, v in by_lang.items()})
+print("test", len(test), collections.Counter(lang[i] for i in test))
diff --git a/ix-cli/scripts/ranking-eval/picks-issue.mjs b/ix-cli/scripts/ranking-eval/picks-issue.mjs
new file mode 100644
index 00000000..c22cd011
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/picks-issue.mjs
@@ -0,0 +1,457 @@
+// Copyright 2026 Ix Infrastructure Inc.
+// issue.ts of fix/issue-start-picking (b62b524), the branch this one is stacked on, frozen as a
+// second baseline: its wider isSourcePath and name order, the old BM25. Transpiled; imports repointed.
+import { readFileSync } from "node:fs";
+import { posix } from "node:path";
+import { isTestPath } from "../../dist/cli/explain/related-files.js";
+import { resolveToken } from "../../dist/cli/explain/text-references.js";
+/**
+ * From an issue's text to the files its fix starts in, for
+ * `ix context --from-issue`.
+ *
+ * Measured on SWE-PolyBench (28 fresh instances, not used while building
+ * this): resolving the code names an issue mentions to their definitions put
+ * a changed file in the top five 0.600 of the time, against 0.485 for BM25
+ * over the repository. But a graph-expanded bundle, ranked by graph distance,
+ * lost to BM25 at equal size (0.57 against 0.67). So the ranking here is
+ * lexical first: the resolved starting files lead, BM25 against the issue
+ * orders the rest, and graph closeness to a starting point only breaks ties
+ * and nudges.
+ *
+ * Starting points, most specific first:
+ *
+ * 1. file paths the issue names, resolved against the tracked files;
+ * 2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`
+ * -- from backticks or running text. A multi-part name is almost always a
+ * code name;
+ * 3. single backticked words of four letters or more. `save` was the right
+ * start for a keras fix; `flex`, `code` and `version` were not, which is
+ * why a single word counts only in backticks, and only after the rest.
+ *
+ * A name resolves only to an exact-name definition in source code. The first
+ * pilot run started from a CSS output file, a changelog, test fixtures, and an
+ * import: `borderStylesReset` resolved to the file that imported it, because
+ * the graph's `module` entity for an import is located in the importing file.
+ */
+/** Starting points a bundle is built from. */
+export const MAX_STARTS = 3;
+/** Candidate names looked up, in order. */
+export const MAX_CANDIDATES = 25;
+/** Files larger than this are not scored: generated, bundled or data. */
+export const MAX_BM25_BYTES = 200 * 1024;
+/** Ranked files carried in the JSON bundle. */
+export const MAX_RANKED_FILES = 20;
+/**
+ * How much being next to a starting point is worth, as a fraction of a file's
+ * BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at
+ * equal size, so closeness may reorder files the text scores alike and may
+ * not overturn a clear lexical lead.
+ */
+export const CLOSENESS_BOOST = 0.1;
+/** Extensions of files a fix edits. */
+const CODE_EXTENSIONS = [
+ "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",
+ "kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",
+];
+const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);
+/**
+ * Not code a fix edits: tests, test data, samples, docs, vendored or built
+ * output. The pilot's list, plus what `isTestPath` already knows.
+ *
+ * The second line is a project's website, playground, test configuration,
+ * benchmarks, stories and end-to-end suites. Svelte's `site/` alone was 40
+ * of the wrong files in Ix's top-10 rankings over two samples of issues; none
+ * of these directories holds any of the 932 files the 382 SWE-PolyBench
+ * Verified fixes change.
+ */
+const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"
+ + "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased|"
+ + "site|website|docs-site|playground|tests?[_-]config|benchmarks|\\.?storybook|e2e|cypress|integration-tests)/"
+ + "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");
+/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */
+export function isSourcePath(path) {
+ return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);
+}
+// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,
+// and the absolute path of a stack trace frame or the tail of a repository URL,
+// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which
+// `resolveIssuePath` cuts down to the tracked path they end in.
+//
+// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then
+// only begin where a run of path characters begins, so a long unbroken run --
+// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.
+const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");
+const BACKTICK = /`([^`\n]{2,80})`/g;
+const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;
+// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.
+const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";
+const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");
+const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);
+/** The paths and code names an issue mentions, most specific first. */
+export function extractCandidates(text) {
+ const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));
+ // A path's own parts are not names in the issue: `saving_api` in
+ // `saving/saving_api.py` is the path again, not a function.
+ const prose = text.replace(PATH, " ");
+ const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));
+ const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));
+ const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);
+ // Only MAX_STARTS names become starts, so order decides: a short common
+ // word (`repeat`, `ignore`) mentioned first took a slot a specific one
+ // (`Stylesheet`) mentioned later needed. Common words still resolve, last.
+ const ordered = [
+ ...multi,
+ ...single.filter((t) => !isGenericIdentifier(t)),
+ ...single.filter((t) => isGenericIdentifier(t)),
+ ];
+ const seen = new Set(paths);
+ const identifiers = [];
+ for (const token of ordered) {
+ if (seen.has(token))
+ continue;
+ seen.add(token);
+ identifiers.push(token);
+ }
+ return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };
+}
+/** Kinds a definition has, most navigable first. Anything else ranks after. */
+const DEFINITION_KINDS = [
+ "class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",
+ "constant", "variable", "property", "field",
+];
+/** Never a definition: an import (located in the importer), a chunk, a file. */
+const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);
+function kindRank(kind) {
+ const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());
+ return i === -1 ? DEFINITION_KINDS.length : i;
+}
+/**
+ * Up to `max` starting points, first found wins, one per file; and the
+ * candidates that resolved to nothing. Candidates after the last start are
+ * not looked up, and not reported as unresolved.
+ */
+export async function pickStartingPoints(text, deps, max = MAX_STARTS) {
+ const { paths, identifiers } = extractCandidates(text);
+ const starts = [];
+ const unresolved = [];
+ const taken = (path) => starts.some((s) => s.path === path);
+ const tracked = new Set(deps.files);
+ const byBasename = new Map();
+ for (const file of deps.files) {
+ const base = posix.basename(file);
+ byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
+ }
+ for (const token of paths) {
+ if (starts.length >= max)
+ return { starts, unresolved };
+ const path = resolveIssuePath(token, tracked, byBasename);
+ if (!path || !isSourcePath(path)) {
+ unresolved.push(token);
+ continue;
+ }
+ if (!taken(path))
+ starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });
+ }
+ const score = deps.fileScore ?? (() => 0);
+ for (const token of identifiers) {
+ if (starts.length >= max)
+ break;
+ // Not caught: a backend that cannot answer is an error to report, not an
+ // issue that names nothing.
+ const hits = (await deps.search(token))
+ .filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))
+ .sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));
+ if (hits.length === 0) {
+ unresolved.push(token);
+ continue;
+ }
+ const best = hits[0];
+ if (taken(best.path))
+ continue;
+ starts.push({
+ token, id: best.id, name: best.name, kind: best.kind, path: best.path,
+ ...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),
+ ...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),
+ via: "identifier in issue",
+ });
+ }
+ return { starts, unresolved };
+}
+/**
+ * A path as an issue writes it, to the tracked file it names. First as
+ * `resolveToken` reads a path in code; then, for an absolute path from a
+ * stack trace or the tail of a repository URL, the longest of its suffixes
+ * that is a tracked path: `/home/u/proj/src/a.py` and
+ * `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.
+ */
+function resolveIssuePath(token, tracked, byBasename) {
+ const direct = resolveToken(token, "", tracked, byBasename);
+ if (direct)
+ return direct;
+ const parts = token.split("/");
+ for (let i = 1; i < parts.length; i++) {
+ const suffix = parts.slice(i).join("/");
+ if (tracked.has(suffix))
+ return suffix;
+ }
+ return undefined;
+}
+/**
+ * Words for BM25: identifiers split at camelCase and underscores, lowercased,
+ * parts of two letters or fewer dropped. `listByKind` is `list kind`, so an
+ * issue that says "list by kind" matches the code that spells it as one word.
+ */
+export function bm25Tokens(text) {
+ const out = [];
+ for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {
+ const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];
+ for (const part of parts)
+ if (part.length > 2)
+ out.push(part.toLowerCase());
+ }
+ return out;
+}
+/**
+ * Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files
+ * that share no word with it left out. A file's path is part of its text, so
+ * `auth/login.ts` matches an issue about logging in before it is opened.
+ */
+export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {
+ // Only the query's words are counted: BM25 needs no other term frequency,
+ // and a map of every word of every file is most of the memory and a third
+ // of the time on a large repository. A document's length is still every word.
+ const terms = [...new Set(bm25Tokens(query))];
+ const wanted = new Set(terms);
+ const docs = new Map();
+ for (const path of files) {
+ const text = repo.read(path);
+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)
+ continue;
+ const tf = new Map();
+ const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);
+ for (const t of tokens)
+ if (wanted.has(t))
+ tf.set(t, (tf.get(t) ?? 0) + 1);
+ docs.set(path, { tf, length: tokens.length });
+ }
+ if (docs.size === 0)
+ return [];
+ const n = docs.size;
+ let total = 0;
+ const df = new Map();
+ for (const doc of docs.values()) {
+ total += doc.length;
+ for (const t of doc.tf.keys())
+ df.set(t, (df.get(t) ?? 0) + 1);
+ }
+ const avg = total / n || 1;
+ const out = [];
+ for (const [path, doc] of docs) {
+ let score = 0;
+ for (const t of terms) {
+ const f = doc.tf.get(t);
+ if (!f)
+ continue;
+ const d = df.get(t);
+ const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));
+ score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));
+ }
+ if (score > 0)
+ out.push({ path, score });
+ }
+ return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));
+}
+/**
+ * Starting points and the lexical ranking for one issue. Without a git
+ * checkout there is no file list: paths do not resolve, BM25 is empty, and
+ * only names can start the bundle.
+ */
+export async function planIssue(text, deps) {
+ const files = deps.repo?.files() ?? [];
+ const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];
+ const scores = new Map(bm25.map((h) => [h.path, h.score]));
+ const { starts, unresolved } = await pickStartingPoints(text, {
+ files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,
+ });
+ if (starts.length > 0 || bm25.length === 0)
+ return { starts, unresolved, fallback: false, bm25 };
+ const top = bm25[0].path;
+ return {
+ starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],
+ unresolved,
+ fallback: true,
+ bm25,
+ };
+}
+/**
+ * Whether an identifier the issue names is too common to point anywhere: a
+ * short all-lowercase word such as `debug`, `bind`, `config` or `render`
+ * resolves to *a* definition, rarely the one the fix touches. Specific names
+ * carry an inner capital (`toJsonTree`, `JsonWriter`), an underscore
+ * (`get_openai_callback`), or length.
+ *
+ * Fixed before it was scored on held-out samples: on SWE-PolyBench issues the
+ * starts it keeps were in the gold patch 20 times in 32, the ones it drops 2
+ * in 7.
+ */
+export function isGenericIdentifier(token) {
+ return !(/[a-z][A-Z]/.test(token) || /^[A-Z][a-z]+[A-Z]/.test(token) || token.includes("_") || token.length >= 8);
+}
+/**
+ * A plan is confident when the issue named a path, or a specific identifier,
+ * that resolved. A BM25 fallback never is: across four samples of
+ * SWE-PolyBench issues its start was in the gold patch once in 55.
+ */
+export function issueConfidence(plan) {
+ if (plan.fallback) {
+ return { confident: false, starts: [], reason: "no code name in the issue resolved to a definition" };
+ }
+ const starts = plan.starts.filter((s) => s.via === "path in issue"
+ || (s.via === "identifier in issue" && !isGenericIdentifier(s.token)));
+ if (starts.length === 0) {
+ const names = plan.starts.map((s) => s.token);
+ return {
+ confident: false,
+ starts,
+ reason: names.length > 0
+ ? `only common words resolved (${names.join(", ")}), which name many definitions`
+ : "nothing in the issue resolved to a definition",
+ };
+ }
+ return { confident: true, starts };
+}
+/** Files after the starting points in a lean view. */
+export const LEAN_RANKED = 4;
+/**
+ * The lean view of a plan. A full bundle costs about 1.5k tokens, and an
+ * agent re-reads it on every turn; on SWE-PolyBench it added a median 14%
+ * tokens even where its start was right, and saved no turns. What the agent
+ * uses is the pointer, so that is all this keeps: the trusted starting points
+ * and a few ranked files -- Ix's starts, then BM25, the ordering that beat
+ * both alone on held-out issues. When nothing is trusted it says so, rather
+ * than pointing the agent at a guess.
+ */
+export function leanIssueView(plan, ranked = LEAN_RANKED) {
+ const c = issueConfidence(plan);
+ if (!c.confident) {
+ return { confidence: "low", reason: c.reason, startingPoints: [], alsoRanked: [], unresolved: plan.unresolved };
+ }
+ const startPaths = new Set(c.starts.map((s) => s.path));
+ const alsoRanked = rankIssueFiles({ starts: [], bm25: plan.bm25, near: new Map() }, ranked + startPaths.size)
+ .filter((f) => !startPaths.has(f.path))
+ .slice(0, ranked);
+ return { confidence: "high", startingPoints: c.starts, alsoRanked, unresolved: plan.unresolved };
+}
+/** BM25 files tried, best first, for one the graph has a node for. */
+export const CENTRE_FALLBACK_TRIES = 10;
+/**
+ * The starting point a bundle is centred on: the first start the graph has a
+ * node for, else the best BM25 file that has one.
+ *
+ * A start can have no node: the graph does not index every tracked file.
+ * On SWE-PolyBench two svelte issues named no code, BM25's top file was a
+ * `.svelte` component, and the command failed with no bundle at all while
+ * the fifth-ranked file was a perfectly good `.js` module. Walking BM25's
+ * ranking finds that one; the files the walk skipped still rank as usual.
+ * `walked` says the centre came from this walk, which is a fallback too.
+ */
+export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {
+ const located = await Promise.all(starts.map(async (s) => {
+ const id = s.id ?? (await findNode(s.path));
+ return id ? { ...s, id } : s;
+ }));
+ const found = located.find((s) => s.id);
+ if (found)
+ return { starts: located, centre: found, walked: false };
+ const tried = new Set(located.map((s) => s.path));
+ for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {
+ const id = await findNode(hit.path);
+ if (!id)
+ continue;
+ const centre = {
+ token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",
+ };
+ return { starts: [...located, centre], centre, walked: true };
+ }
+ return { starts: located, walked: false };
+}
+/**
+ * The files to read, best first: the starting files in the order they were
+ * found, then every other source file by
+ *
+ * bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))
+ *
+ * where closeness is 1 for a file one hop from a starting point, 0.5 for one
+ * of its ranked related files, 0 otherwise. Ties go to the closer file, then
+ * the path. A graph neighbour the issue's text does not match at all still
+ * ranks, after every file it does match.
+ */
+export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {
+ const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));
+ const out = [];
+ const startPaths = new Set();
+ for (const s of input.starts) {
+ if (startPaths.has(s.path))
+ continue;
+ startPaths.add(s.path);
+ out.push({
+ path: s.path,
+ score: round(lexical.get(s.path) ?? 0),
+ reason: `starting point (${s.via}: ${s.token})`,
+ });
+ }
+ const pool = new Set([...lexical.keys(), ...input.near.keys()]);
+ const rest = [...pool]
+ .filter((path) => !startPaths.has(path) && isSourcePath(path))
+ .map((path) => {
+ const bm25 = lexical.get(path) ?? 0;
+ const near = input.near.get(path);
+ const closeness = near?.weight ?? 0;
+ const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]
+ .filter(Boolean).join("; ");
+ return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };
+ })
+ .sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));
+ for (const r of rest)
+ out.push({ path: r.path, score: round(r.score), reason: r.reason });
+ return out.slice(0, limit);
+}
+/**
+ * The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the
+ * stream can be replaced under test.
+ */
+export async function readIssueText(arg, stdin = process.stdin) {
+ if (arg !== "-")
+ return decodeIssue(readFileSync(arg));
+ const chunks = [];
+ for await (const chunk of stdin) {
+ chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));
+ }
+ return decodeIssue(Buffer.concat(chunks));
+}
+/**
+ * UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`
+ * writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >
+ * issue.md` there produces a file that, read as UTF-8, is every letter
+ * followed by a NUL: no name, no path, and no BM25 word in it.
+ */
+function decodeIssue(bytes) {
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
+ return bytes.subarray(2).toString("utf16le");
+ }
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+ const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));
+ return swapped.swap16().toString("utf16le");
+ }
+ const text = bytes.toString("utf8");
+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+}
+function unique(items) {
+ return [...new Set(items)];
+}
+function round(n) {
+ return Math.round(n * 1000) / 1000;
+}
+function cmp(a, b) {
+ return a < b ? -1 : a > b ? 1 : 0;
+}
diff --git a/ix-cli/scripts/ranking-eval/results/README.md b/ix-cli/scripts/ranking-eval/results/README.md
new file mode 100644
index 00000000..2d7e4477
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/README.md
@@ -0,0 +1,3 @@
+Summary tables of the ranking study. The per-issue records they were computed
+from (`*.jsonl`, about 5 MB) are kept out of this repository, in ix-bench's
+`results/ranking-eval/`. `eval.mjs` writes new ones here; don't commit them.
diff --git a/ix-cli/scripts/ranking-eval/results/dev-ablation.txt b/ix-cli/scripts/ranking-eval/results/dev-ablation.txt
new file mode 100644
index 00000000..1303f8cf
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/dev-ablation.txt
@@ -0,0 +1,17 @@
+metric=bm25 ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+final | 0.600 | 0.737 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.73/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.74/0.75 (12) | 25/9 | 24/5 | 23/3
+final_nocode | 0.569 | 0.704 | 0.814 | 0.64/0.75/0.80 (26) | 0.56/0.72/0.88 (44) | 0.54/0.63/0.74 (22) | 0.51/0.67/0.72 (12) | 23/11 | 17/5 | 18/2
+final_nostop | 0.557 | 0.697 | 0.784 | 0.66/0.72/0.75 (26) | 0.54/0.70/0.82 (44) | 0.49/0.67/0.80 (22) | 0.53/0.67/0.72 (12) | 19/7 | 15/2 | 14/2
+final_noid | 0.603 | 0.727 | 0.817 | 0.69/0.75/0.81 (26) | 0.59/0.78/0.88 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 23/7 | 19/4 | 17/1
+final_nosite | 0.579 | 0.725 | 0.829 | 0.70/0.75/0.82 (26) | 0.53/0.71/0.87 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/10 | 22/6 | 21/3
+
+metric=plan ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+final | 0.632 | 0.761 | 0.848 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.89 (44) | 0.80/0.84/0.84 (22) | 0.56/0.74/0.75 (12) | 17/8 | 16/5 | 16/3
+final_nocode | 0.639 | 0.753 | 0.836 | 0.68/0.76/0.81 (26) | 0.56/0.72/0.88 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 16/7 | 12/4 | 13/2
+final_nostop | 0.603 | 0.733 | 0.793 | 0.64/0.72/0.75 (26) | 0.52/0.70/0.82 (44) | 0.75/0.84/0.84 (22) | 0.53/0.67/0.72 (12) | 10/5 | 9/2 | 8/2
+final_noid | 0.620 | 0.744 | 0.836 | 0.64/0.74/0.81 (26) | 0.56/0.75/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 14/7 | 11/5 | 11/1
+final_nosite | 0.621 | 0.749 | 0.829 | 0.68/0.75/0.82 (26) | 0.51/0.71/0.85 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/9 | 14/6 | 14/3
diff --git a/ix-cli/scripts/ranking-eval/results/dev-r1.txt b/ix-cli/scripts/ranking-eval/results/dev-r1.txt
new file mode 100644
index 00000000..fbdb7024
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/dev-r1.txt
@@ -0,0 +1,55 @@
+metric=bm25 ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+code | 0.559 | 0.646 | 0.762 | 0.61/0.69/0.73 (26) | 0.58/0.70/0.83 (44) | 0.52/0.54/0.70 (22) | 0.45/0.56/0.69 (12) | 13/1 | 6/1 | 8/1
+stop | 0.540 | 0.691 | 0.786 | 0.61/0.69/0.80 (26) | 0.54/0.75/0.87 (44) | 0.47/0.61/0.65 (22) | 0.51/0.61/0.70 (12) | 15/8 | 13/4 | 13/2
+min4 | 0.494 | 0.607 | 0.732 | 0.42/0.59/0.70 (26) | 0.55/0.68/0.81 (44) | 0.45/0.52/0.63 (22) | 0.53/0.56/0.72 (12) | 7/7 | 2/4 | 3/2
+qtf | 0.535 | 0.665 | 0.726 | 0.52/0.65/0.69 (26) | 0.55/0.73/0.81 (44) | 0.52/0.61/0.63 (22) | 0.53/0.56/0.69 (12) | 11/5 | 7/1 | 5/4
+qtf8 | 0.535 | 0.664 | 0.764 | 0.54/0.61/0.78 (26) | 0.53/0.74/0.83 (44) | 0.52/0.63/0.66 (22) | 0.56/0.56/0.69 (12) | 19/10 | 12/4 | 12/3
+id2 | 0.555 | 0.670 | 0.728 | 0.56/0.67/0.71 (26) | 0.58/0.72/0.80 (44) | 0.52/0.58/0.63 (22) | 0.53/0.65/0.69 (12) | 17/5 | 9/1 | 4/1
+id3 | 0.534 | 0.652 | 0.757 | 0.56/0.68/0.76 (26) | 0.52/0.66/0.81 (44) | 0.52/0.58/0.67 (22) | 0.53/0.69/0.72 (12) | 17/9 | 12/4 | 8/1
+id5 | 0.527 | 0.671 | 0.765 | 0.54/0.71/0.77 (26) | 0.53/0.68/0.80 (44) | 0.47/0.58/0.72 (22) | 0.59/0.72/0.72 (12) | 20/11 | 15/6 | 13/6
+id3p | 0.519 | 0.637 | 0.741 | 0.56/0.68/0.76 (26) | 0.51/0.64/0.79 (44) | 0.47/0.56/0.63 (22) | 0.53/0.69/0.72 (12) | 16/10 | 11/5 | 7/3
+path1 | 0.496 | 0.617 | 0.730 | 0.45/0.61/0.71 (26) | 0.58/0.69/0.82 (44) | 0.45/0.52/0.61 (22) | 0.40/0.56/0.69 (12) | 2/1 | 0/1 | 1/0
+path2 | 0.505 | 0.612 | 0.728 | 0.47/0.61/0.70 (26) | 0.58/0.68/0.82 (44) | 0.45/0.52/0.61 (22) | 0.40/0.56/0.69 (12) | 3/1 | 0/2 | 0/0
+path4 | 0.507 | 0.617 | 0.728 | 0.47/0.61/0.70 (26) | 0.58/0.69/0.82 (44) | 0.45/0.52/0.61 (22) | 0.42/0.56/0.69 (12) | 4/1 | 0/1 | 0/0
+path2base | 0.496 | 0.635 | 0.739 | 0.45/0.61/0.70 (26) | 0.58/0.70/0.84 (44) | 0.45/0.52/0.61 (22) | 0.40/0.65/0.69 (12) | 2/1 | 3/2 | 2/0
+path4base | 0.496 | 0.635 | 0.739 | 0.45/0.61/0.70 (26) | 0.58/0.70/0.84 (44) | 0.45/0.52/0.61 (22) | 0.40/0.65/0.69 (12) | 2/1 | 3/2 | 2/0
+k1_09 | 0.493 | 0.611 | 0.717 | 0.42/0.56/0.70 (26) | 0.59/0.72/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.51/0.69 (12) | 1/4 | 1/5 | 1/3
+k1_15 | 0.520 | 0.615 | 0.717 | 0.51/0.61/0.69 (26) | 0.58/0.68/0.82 (44) | 0.47/0.52/0.56 (22) | 0.40/0.56/0.69 (12) | 6/2 | 2/3 | 1/2
+k1_20 | 0.526 | 0.626 | 0.726 | 0.53/0.63/0.70 (26) | 0.57/0.69/0.81 (44) | 0.42/0.52/0.61 (22) | 0.53/0.56/0.72 (12) | 10/4 | 5/3 | 5/3
+b03 | 0.506 | 0.616 | 0.721 | 0.45/0.54/0.67 (26) | 0.60/0.69/0.82 (44) | 0.47/0.61/0.61 (22) | 0.33/0.51/0.69 (12) | 7/7 | 3/7 | 1/3
+b05 | 0.501 | 0.616 | 0.725 | 0.43/0.57/0.68 (26) | 0.61/0.70/0.82 (44) | 0.45/0.56/0.61 (22) | 0.33/0.51/0.69 (12) | 3/5 | 3/6 | 1/1
+b09 | 0.503 | 0.606 | 0.729 | 0.48/0.61/0.69 (26) | 0.56/0.66/0.82 (44) | 0.47/0.52/0.61 (22) | 0.42/0.56/0.72 (12) | 6/3 | 2/4 | 4/3
+cap3 | 0.506 | 0.611 | 0.723 | 0.41/0.53/0.62 (26) | 0.58/0.69/0.84 (44) | 0.54/0.61/0.65 (22) | 0.38/0.49/0.63 (12) | 8/8 | 3/9 | 3/5
+max100 | 0.473 | 0.595 | 0.691 | 0.43/0.62/0.71 (26) | 0.57/0.68/0.79 (44) | 0.38/0.42/0.47 (22) | 0.38/0.56/0.69 (12) | 2/3 | 1/3 | 1/4
+nosite | 0.509 | 0.646 | 0.747 | 0.45/0.61/0.70 (26) | 0.60/0.73/0.82 (44) | 0.47/0.56/0.70 (22) | 0.38/0.56/0.69 (12) | 3/0 | 3/0 | 3/0
+nodts | 0.497 | 0.619 | 0.726 | 0.43/0.61/0.70 (26) | 0.59/0.69/0.81 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 0/1 | 0/1
+
+metric=plan ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+code | 0.621 | 0.709 | 0.792 | 0.66/0.68/0.74 (26) | 0.58/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 8/1 | 2/1 | 4/2
+stop | 0.592 | 0.743 | 0.829 | 0.64/0.75/0.81 (26) | 0.52/0.72/0.87 (44) | 0.71/0.84/0.84 (22) | 0.53/0.63/0.72 (12) | 9/8 | 10/4 | 9/1
+min4 | 0.584 | 0.694 | 0.780 | 0.59/0.70/0.75 (26) | 0.56/0.68/0.81 (44) | 0.66/0.80/0.80 (22) | 0.53/0.56/0.72 (12) | 5/5 | 2/2 | 2/2
+qtf | 0.594 | 0.719 | 0.773 | 0.64/0.70/0.74 (26) | 0.53/0.73/0.81 (44) | 0.71/0.80/0.80 (22) | 0.53/0.56/0.69 (12) | 7/4 | 4/1 | 3/4
+qtf8 | 0.625 | 0.735 | 0.799 | 0.64/0.72/0.77 (26) | 0.55/0.74/0.83 (44) | 0.80/0.84/0.84 (22) | 0.56/0.56/0.69 (12) | 13/6 | 9/3 | 9/4
+id2 | 0.629 | 0.732 | 0.781 | 0.65/0.70/0.74 (26) | 0.56/0.72/0.80 (44) | 0.80/0.84/0.84 (22) | 0.53/0.65/0.69 (12) | 12/4 | 5/1 | 3/2
+id3 | 0.623 | 0.714 | 0.787 | 0.64/0.71/0.75 (26) | 0.55/0.66/0.80 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 11/6 | 8/4 | 5/2
+id5 | 0.626 | 0.725 | 0.795 | 0.67/0.71/0.76 (26) | 0.53/0.68/0.79 (44) | 0.78/0.84/0.89 (22) | 0.59/0.72/0.72 (12) | 16/8 | 11/5 | 10/7
+id3p | 0.618 | 0.705 | 0.781 | 0.64/0.71/0.75 (26) | 0.54/0.64/0.79 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 11/7 | 8/5 | 5/4
+path1 | 0.584 | 0.690 | 0.778 | 0.60/0.68/0.74 (26) | 0.58/0.68/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 0/2 | 0/1
+path2 | 0.581 | 0.690 | 0.775 | 0.59/0.68/0.74 (26) | 0.58/0.68/0.81 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/2 | 0/2
+path4 | 0.581 | 0.695 | 0.778 | 0.59/0.68/0.74 (26) | 0.58/0.69/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/1 | 0/1
+path2base | 0.585 | 0.711 | 0.789 | 0.61/0.69/0.74 (26) | 0.58/0.70/0.84 (44) | 0.66/0.80/0.80 (22) | 0.42/0.65/0.69 (12) | 2/1 | 3/2 | 2/1
+path4base | 0.585 | 0.711 | 0.789 | 0.61/0.69/0.74 (26) | 0.58/0.70/0.84 (44) | 0.66/0.80/0.80 (22) | 0.42/0.65/0.69 (12) | 2/1 | 3/2 | 2/1
+k1_09 | 0.586 | 0.695 | 0.770 | 0.58/0.68/0.75 (26) | 0.60/0.70/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.53/0.69 (12) | 1/2 | 0/2 | 1/3
+k1_15 | 0.607 | 0.702 | 0.787 | 0.64/0.70/0.74 (26) | 0.58/0.67/0.82 (44) | 0.71/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 4/0 | 3/3 | 1/1
+k1_20 | 0.611 | 0.707 | 0.787 | 0.64/0.70/0.74 (26) | 0.57/0.68/0.81 (44) | 0.71/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 6/2 | 4/3 | 3/2
+b03 | 0.610 | 0.695 | 0.775 | 0.61/0.68/0.74 (26) | 0.60/0.70/0.82 (44) | 0.75/0.80/0.80 (22) | 0.38/0.53/0.66 (12) | 7/6 | 1/4 | 1/3
+b05 | 0.593 | 0.697 | 0.783 | 0.61/0.69/0.75 (26) | 0.61/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.38/0.53/0.69 (12) | 4/4 | 2/4 | 1/0
+b09 | 0.594 | 0.692 | 0.787 | 0.64/0.70/0.73 (26) | 0.56/0.65/0.81 (44) | 0.71/0.84/0.84 (22) | 0.42/0.56/0.72 (12) | 4/2 | 3/4 | 2/3
+cap3 | 0.620 | 0.696 | 0.775 | 0.61/0.69/0.73 (26) | 0.62/0.69/0.82 (44) | 0.75/0.80/0.80 (22) | 0.42/0.53/0.66 (12) | 8/5 | 2/4 | 2/3
+max100 | 0.562 | 0.674 | 0.752 | 0.59/0.69/0.75 (26) | 0.56/0.68/0.79 (44) | 0.62/0.71/0.71 (22) | 0.42/0.56/0.69 (12) | 0/2 | 1/3 | 0/3
+nosite | 0.586 | 0.715 | 0.781 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/0
+nodts | 0.581 | 0.698 | 0.779 | 0.59/0.68/0.75 (26) | 0.58/0.69/0.81 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/1 | 0/1
diff --git a/ix-cli/scripts/ranking-eval/results/dev-r2.txt b/ix-cli/scripts/ranking-eval/results/dev-r2.txt
new file mode 100644
index 00000000..f4774107
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/dev-r2.txt
@@ -0,0 +1,37 @@
+metric=bm25 ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+code | 0.559 | 0.646 | 0.762 | 0.61/0.69/0.73 (26) | 0.58/0.70/0.83 (44) | 0.52/0.54/0.70 (22) | 0.45/0.56/0.69 (12) | 13/1 | 6/1 | 8/1
+stop | 0.540 | 0.691 | 0.786 | 0.61/0.69/0.80 (26) | 0.54/0.75/0.87 (44) | 0.47/0.61/0.65 (22) | 0.51/0.61/0.70 (12) | 15/8 | 13/4 | 13/2
+cs | 0.587 | 0.708 | 0.815 | 0.69/0.75/0.81 (26) | 0.57/0.73/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 20/8 | 17/5 | 16/1
+cs_q8 | 0.580 | 0.726 | 0.781 | 0.67/0.75/0.80 (26) | 0.54/0.73/0.81 (44) | 0.55/0.71/0.71 (22) | 0.58/0.68/0.77 (12) | 28/16 | 26/7 | 19/7
+cs_id2 | 0.579 | 0.725 | 0.829 | 0.70/0.75/0.82 (26) | 0.53/0.71/0.87 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/10 | 22/6 | 21/3
+cs_id3 | 0.598 | 0.720 | 0.828 | 0.70/0.75/0.81 (26) | 0.57/0.69/0.88 (44) | 0.54/0.73/0.80 (22) | 0.61/0.74/0.75 (12) | 26/11 | 23/6 | 21/4
+cs_q8_id2 | 0.580 | 0.724 | 0.823 | 0.69/0.78/0.83 (26) | 0.53/0.71/0.81 (44) | 0.55/0.68/0.85 (22) | 0.60/0.71/0.80 (12) | 27/14 | 26/8 | 24/7
+cs_site | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1
+cs_k15 | 0.573 | 0.702 | 0.811 | 0.69/0.76/0.81 (26) | 0.54/0.71/0.87 (44) | 0.54/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 20/9 | 16/5 | 18/2
+cs_k20 | 0.544 | 0.707 | 0.824 | 0.70/0.77/0.82 (26) | 0.48/0.71/0.87 (44) | 0.49/0.67/0.80 (22) | 0.51/0.63/0.72 (12) | 21/12 | 17/5 | 20/2
+cs_big05 | 0.569 | 0.712 | 0.812 | 0.65/0.76/0.81 (26) | 0.58/0.74/0.88 (44) | 0.49/0.67/0.74 (22) | 0.51/0.58/0.69 (12) | 19/8 | 17/4 | 15/1
+cs_big1 | 0.558 | 0.708 | 0.813 | 0.65/0.76/0.81 (26) | 0.56/0.74/0.88 (44) | 0.49/0.67/0.74 (22) | 0.47/0.54/0.69 (12) | 19/9 | 16/4 | 15/1
+cs_path2base | 0.589 | 0.723 | 0.809 | 0.69/0.75/0.81 (26) | 0.57/0.75/0.87 (44) | 0.54/0.67/0.74 (22) | 0.51/0.67/0.70 (12) | 21/8 | 18/4 | 16/2
+max256 | 0.498 | 0.632 | 0.739 | 0.43/0.61/0.70 (26) | 0.56/0.69/0.81 (44) | 0.51/0.58/0.67 (22) | 0.38/0.56/0.69 (12) | 2/1 | 2/1 | 2/1
+code_big1 | 0.560 | 0.658 | 0.763 | 0.63/0.70/0.73 (26) | 0.57/0.70/0.83 (44) | 0.52/0.58/0.70 (22) | 0.45/0.56/0.69 (12) | 16/3 | 7/0 | 9/1
+
+metric=plan ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+code | 0.621 | 0.709 | 0.792 | 0.66/0.68/0.74 (26) | 0.58/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 8/1 | 2/1 | 4/2
+stop | 0.592 | 0.743 | 0.829 | 0.64/0.75/0.81 (26) | 0.52/0.72/0.87 (44) | 0.71/0.84/0.84 (22) | 0.53/0.63/0.72 (12) | 9/8 | 10/4 | 9/1
+cs | 0.613 | 0.725 | 0.836 | 0.64/0.74/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 12/8 | 9/6 | 11/1
+cs_q8 | 0.635 | 0.756 | 0.809 | 0.70/0.76/0.80 (26) | 0.53/0.73/0.81 (44) | 0.80/0.84/0.84 (22) | 0.58/0.68/0.76 (12) | 19/12 | 19/7 | 14/7
+cs_id2 | 0.621 | 0.749 | 0.829 | 0.68/0.75/0.82 (26) | 0.51/0.71/0.85 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/9 | 14/6 | 14/3
+cs_id3 | 0.627 | 0.742 | 0.828 | 0.70/0.75/0.81 (26) | 0.53/0.69/0.85 (44) | 0.78/0.84/0.84 (22) | 0.53/0.74/0.75 (12) | 19/11 | 16/7 | 15/5
+cs_q8_id2 | 0.637 | 0.748 | 0.832 | 0.73/0.78/0.83 (26) | 0.51/0.71/0.81 (44) | 0.80/0.80/0.90 (22) | 0.60/0.71/0.80 (12) | 21/11 | 19/8 | 19/7
+cs_site | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1
+cs_k15 | 0.590 | 0.723 | 0.829 | 0.65/0.75/0.81 (26) | 0.49/0.69/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 11/9 | 7/4 | 11/2
+cs_k20 | 0.585 | 0.730 | 0.833 | 0.70/0.75/0.82 (26) | 0.45/0.70/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.59/0.72 (12) | 13/10 | 9/6 | 12/2
+cs_big05 | 0.617 | 0.729 | 0.833 | 0.65/0.75/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 14/8 | 9/5 | 10/1
+cs_big1 | 0.605 | 0.729 | 0.834 | 0.65/0.75/0.81 (26) | 0.53/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.49/0.56/0.69 (12) | 14/9 | 9/5 | 10/1
+cs_path2base | 0.613 | 0.737 | 0.829 | 0.64/0.74/0.81 (26) | 0.55/0.71/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.65/0.69 (12) | 12/8 | 10/6 | 10/2
+max256 | 0.584 | 0.703 | 0.782 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.81 (44) | 0.68/0.81/0.81 (22) | 0.42/0.56/0.69 (12) | 1/0 | 1/0 | 1/1
+code_big1 | 0.600 | 0.711 | 0.793 | 0.62/0.69/0.74 (26) | 0.55/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 9/3 | 2/0 | 5/2
diff --git a/ix-cli/scripts/ranking-eval/results/dev-r3.txt b/ix-cli/scripts/ranking-eval/results/dev-r3.txt
new file mode 100644
index 00000000..641d4c31
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/dev-r3.txt
@@ -0,0 +1,33 @@
+metric=bm25 ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+cs | 0.587 | 0.708 | 0.815 | 0.69/0.75/0.81 (26) | 0.57/0.73/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 20/8 | 17/5 | 16/1
+cs_site | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1
+cs_site0 | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1
+cs_siteonly | 0.603 | 0.727 | 0.824 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 23/7 | 19/4 | 17/1
+css_id2 | 0.633 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.64/0.76/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 29/9 | 24/5 | 23/3
+css_id3 | 0.629 | 0.751 | 0.837 | 0.70/0.75/0.81 (26) | 0.61/0.77/0.90 (44) | 0.59/0.73/0.80 (22) | 0.61/0.74/0.75 (12) | 30/10 | 26/5 | 22/4
+css_q8 | 0.604 | 0.736 | 0.791 | 0.67/0.75/0.80 (26) | 0.58/0.75/0.83 (44) | 0.59/0.71/0.71 (22) | 0.58/0.68/0.77 (12) | 31/16 | 27/7 | 19/6
+css_q2 | 0.624 | 0.732 | 0.826 | 0.71/0.75/0.82 (26) | 0.60/0.76/0.88 (44) | 0.64/0.68/0.75 (22) | 0.52/0.66/0.77 (12) | 28/10 | 24/4 | 23/3
+css_k15 | 0.609 | 0.724 | 0.821 | 0.69/0.76/0.81 (26) | 0.60/0.77/0.89 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 25/8 | 18/3 | 19/2
+css_k10 | 0.614 | 0.739 | 0.823 | 0.68/0.74/0.81 (26) | 0.61/0.81/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.69 (12) | 23/6 | 20/3 | 17/1
+css_b06 | 0.615 | 0.741 | 0.820 | 0.68/0.73/0.80 (26) | 0.61/0.80/0.91 (44) | 0.63/0.72/0.74 (22) | 0.47/0.58/0.69 (12) | 23/7 | 20/4 | 16/1
+css_b09 | 0.609 | 0.730 | 0.817 | 0.69/0.76/0.79 (26) | 0.60/0.77/0.88 (44) | 0.58/0.68/0.75 (22) | 0.51/0.63/0.75 (12) | 25/8 | 19/4 | 19/3
+css_big05 | 0.605 | 0.730 | 0.823 | 0.65/0.76/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.69 (12) | 24/7 | 19/3 | 17/1
+
+metric=plan ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+cs | 0.613 | 0.725 | 0.836 | 0.64/0.74/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 12/8 | 9/6 | 11/1
+cs_site | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1
+cs_site0 | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1
+cs_siteonly | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1
+css_id2 | 0.656 | 0.768 | 0.850 | 0.68/0.75/0.82 (26) | 0.60/0.76/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 20/8 | 16/5 | 17/3
+css_id3 | 0.656 | 0.775 | 0.837 | 0.70/0.75/0.81 (26) | 0.60/0.77/0.88 (44) | 0.78/0.84/0.84 (22) | 0.53/0.74/0.75 (12) | 22/10 | 19/5 | 16/5
+css_q8 | 0.659 | 0.766 | 0.818 | 0.70/0.76/0.80 (26) | 0.59/0.75/0.83 (44) | 0.80/0.84/0.84 (22) | 0.58/0.68/0.76 (12) | 22/12 | 20/7 | 14/6
+css_q2 | 0.632 | 0.764 | 0.846 | 0.70/0.75/0.83 (26) | 0.56/0.76/0.88 (44) | 0.75/0.84/0.84 (22) | 0.52/0.66/0.77 (12) | 18/10 | 16/4 | 18/3
+css_k15 | 0.616 | 0.755 | 0.839 | 0.65/0.75/0.81 (26) | 0.55/0.77/0.89 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/8 | 10/2 | 12/2
+css_k10 | 0.630 | 0.771 | 0.842 | 0.64/0.73/0.81 (26) | 0.59/0.81/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 15/7 | 12/2 | 11/1
+css_b06 | 0.622 | 0.767 | 0.842 | 0.64/0.74/0.81 (26) | 0.58/0.80/0.91 (44) | 0.75/0.84/0.84 (22) | 0.49/0.56/0.69 (12) | 15/9 | 13/3 | 11/1
+css_b09 | 0.616 | 0.763 | 0.836 | 0.65/0.74/0.79 (26) | 0.55/0.77/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.65/0.75 (12) | 15/8 | 11/3 | 13/3
+css_big05 | 0.633 | 0.747 | 0.842 | 0.65/0.75/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 17/7 | 11/4 | 11/1
diff --git a/ix-cli/scripts/ranking-eval/results/dev-v2.txt b/ix-cli/scripts/ranking-eval/results/dev-v2.txt
new file mode 100644
index 00000000..3ba54c20
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/dev-v2.txt
@@ -0,0 +1,32 @@
+metric=bm25 ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+picks | 0.499 | 0.636 | 0.719 | 0.43/0.61/0.70 (26) | 0.60/0.73/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 2/0 | 0/1
+picks_base | 0.499 | 0.636 | 0.719 | 0.43/0.61/0.70 (26) | 0.60/0.73/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 2/0 | 0/1
+final | 0.600 | 0.737 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.73/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.74/0.75 (12) | 25/9 | 24/5 | 23/3
+final2 | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 24/9 | 24/5 | 23/3
+final2_nocode | 0.560 | 0.713 | 0.804 | 0.62/0.75/0.80 (26) | 0.56/0.75/0.88 (44) | 0.52/0.63/0.70 (22) | 0.51/0.67/0.72 (12) | 22/11 | 18/5 | 17/2
+final2_nostop | 0.557 | 0.697 | 0.784 | 0.66/0.72/0.75 (26) | 0.54/0.73/0.82 (44) | 0.49/0.63/0.80 (22) | 0.53/0.67/0.72 (12) | 19/7 | 15/2 | 14/2
+final2_noid | 0.593 | 0.727 | 0.815 | 0.69/0.75/0.81 (26) | 0.59/0.78/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 22/7 | 19/4 | 16/1
+impl | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 24/9 | 24/5 | 23/3
+
+metric=plan ref=base n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0
+picks | 0.586 | 0.715 | 0.771 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/1
+picks_base | 0.586 | 0.715 | 0.771 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/1
+final | 0.632 | 0.761 | 0.848 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.89 (44) | 0.80/0.84/0.84 (22) | 0.56/0.74/0.75 (12) | 17/8 | 16/5 | 16/3
+final2 | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 17/8 | 15/5 | 17/3
+final2_nocode | 0.639 | 0.753 | 0.836 | 0.68/0.76/0.81 (26) | 0.56/0.72/0.88 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 16/7 | 12/4 | 13/2
+final2_nostop | 0.603 | 0.733 | 0.793 | 0.64/0.72/0.75 (26) | 0.52/0.70/0.82 (44) | 0.75/0.84/0.84 (22) | 0.53/0.67/0.72 (12) | 10/5 | 9/2 | 8/2
+final2_noid | 0.620 | 0.744 | 0.836 | 0.64/0.74/0.81 (26) | 0.56/0.75/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 14/7 | 11/5 | 11/1
+impl | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 17/8 | 15/5 | 17/3
+
+metric=bm25 ref=picks n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+final2 | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/9 | 22/5 | 24/3
+impl | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/9 | 22/5 | 24/3
+metric=plan ref=picks n=104
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+final2 | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/8 | 14/6 | 18/3
+impl | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/8 | 14/6 | 18/3
diff --git a/ix-cli/scripts/ranking-eval/results/test-final.txt b/ix-cli/scripts/ranking-eval/results/test-final.txt
new file mode 100644
index 00000000..09a3dde0
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/test-final.txt
@@ -0,0 +1,9 @@
+metric=bm25 ref=base n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.470 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.49/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 0/0 | 0/0 | 0/0
+impl | 0.601 | 0.660 | 0.733 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.76/0.82/0.84 (15) | 0.53/0.57/0.66 (15) | 13/3 | 10/3 | 6/4
+
+metric=plan ref=base n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.517 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.42/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 0/0 | 0/0 | 0/0
+impl | 0.601 | 0.676 | 0.750 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.77/0.82/0.84 (15) | 0.52/0.57/0.66 (15) | 10/4 | 9/2 | 5/4
diff --git a/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txt b/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txt
new file mode 100644
index 00000000..01b136f9
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txt
@@ -0,0 +1,22 @@
+SECOND scoring of the test split, after a post-hoc fix: the first final variant's
+guides|demos?|benchmarks? exclusions dropped files real fixes change, so isSourcePath
+was replaced by fix/issue-start-picking's list. First scoring: test-final.txt.
+
+metric=bm25 ref=base n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.470 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.49/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 0/0 | 0/0 | 0/0
+picks | 0.495 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.59/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 2/0 | 0/0 | 0/0
+impl | 0.606 | 0.671 | 0.744 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.78/0.87/0.89 (15) | 0.53/0.57/0.66 (15) | 14/2 | 11/2 | 7/3
+
+metric=plan ref=base n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+base | 0.517 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.42/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 0/0 | 0/0 | 0/0
+picks | 0.542 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.52/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 2/0 | 0/0 | 0/0
+impl | 0.606 | 0.687 | 0.761 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.79/0.87/0.89 (15) | 0.52/0.57/0.66 (15) | 10/3 | 10/1 | 6/3
+
+metric=bm25 ref=picks n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+impl | 0.606 | 0.671 | 0.744 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.78/0.87/0.89 (15) | 0.53/0.57/0.66 (15) | 13/3 | 11/2 | 7/3
+metric=plan ref=picks n=60
+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20
+impl | 0.606 | 0.687 | 0.761 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.79/0.87/0.89 (15) | 0.52/0.57/0.66 (15) | 9/4 | 10/1 | 6/3
diff --git a/ix-cli/scripts/ranking-eval/speed.mjs b/ix-cli/scripts/ranking-eval/speed.mjs
new file mode 100644
index 00000000..fb524856
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/speed.mjs
@@ -0,0 +1,48 @@
+// Time per issue of the baseline and the current bm25Rank, over the same
+// files and issue text: median of 7 runs after one warm-up.
+//
+// node speed.mjs <instance_id>[,<instance_id>...] [--worktree <dir>]
+//
+// Without --worktree, files are read from memory (the snapshot, preloaded),
+// so the time is tokenising and scoring alone. With --worktree <checkout>,
+// files are read from disk through gitRepoAccess, as `ix context
+// --from-issue` reads them. With --at <owner/repo>@<commit>, the dev issue's
+// text is ranked against that snapshot instead (timing only, no scoring).
+import { readFileSync } from "node:fs";
+
+import * as dist from "../../dist/cli/explain/issue.js";
+import { gitRepoAccess } from "../../dist/cli/explain/text-references.js";
+import * as base from "./baseline-issue.mjs";
+import { loadCorpus } from "./corpus.mjs";
+
+const argv = process.argv.slice(2);
+const ids = new Set(argv[0].split(","));
+const wtIdx = argv.indexOf("--worktree");
+const worktree = wtIdx >= 0 ? argv[wtIdx + 1] : undefined;
+const atIdx = argv.indexOf("--at");
+const at = atIdx >= 0 ? argv[atIdx + 1].split("@") : undefined;
+const rows = readFileSync("/home/ihock/src/ix-bench/results/external/polybench_verified.jsonl", "utf8")
+ .split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((r) => ids.has(r.instance_id));
+
+function time(fn) {
+ fn();
+ const ms = [];
+ for (let i = 0; i < 7; i++) {
+ const t = process.hrtime.bigint();
+ fn();
+ ms.push(Number(process.hrtime.bigint() - t) / 1e6);
+ }
+ return ms.sort((a, b) => a - b)[3];
+}
+
+for (const row of rows) {
+ const repo = worktree ? gitRepoAccess(worktree) : at ? loadCorpus(at[0], at[1]) : loadCorpus(row.repo, row.base_commit);
+ const files = repo.files();
+ const q = row.problem_statement;
+ const bFiles = files.filter(base.isSourcePath);
+ const nFiles = files.filter(dist.isSourcePath);
+ const tb = time(() => base.bm25Rank(repo, bFiles, q));
+ const tn = time(() => dist.bm25Rank(repo, nFiles, q));
+ console.log(`${row.instance_id}${at ? ` on ${at[0]}` : ""} ${worktree ? "disk" : "memory"} files base=${bFiles.length} new=${nFiles.length} `
+ + `issue=${q.length}ch base ${tb.toFixed(0)} ms new ${tn.toFixed(0)} ms`);
+}
diff --git a/ix-cli/scripts/ranking-eval/split.json b/ix-cli/scripts/ranking-eval/split.json
new file mode 100644
index 00000000..416e98ef
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/split.json
@@ -0,0 +1,171 @@
+{
+ "seed": 20260930,
+ "dev": [
+ "apache__dubbo-3093",
+ "apache__dubbo-3317",
+ "apache__dubbo-3622",
+ "apache__dubbo-3855",
+ "apache__dubbo-4026",
+ "apache__rocketmq-1636",
+ "apache__rocketmq-4122",
+ "apache__rocketmq-5008",
+ "apache__rocketmq-5037",
+ "apache__rocketmq-5062",
+ "apolloconfig__apollo-1894",
+ "apolloconfig__apollo-4207",
+ "apolloconfig__apollo-4464",
+ "apolloconfig__apollo-4568",
+ "coder__code-server-3277",
+ "coder__code-server-4923",
+ "coder__code-server-6278",
+ "google__gson-1703",
+ "google__gson-1904",
+ "google__gson-1989",
+ "google__gson-2060",
+ "google__gson-2071",
+ "google__gson-2134",
+ "google__gson-2158",
+ "google__gson-2167",
+ "google__gson-2337",
+ "google__gson-2475",
+ "google__gson-2479",
+ "google__gson-2498",
+ "keras-team__keras-18553",
+ "keras-team__keras-18871",
+ "keras-team__keras-18975",
+ "keras-team__keras-19190",
+ "keras-team__keras-19201",
+ "keras-team__keras-19459",
+ "keras-team__keras-19466",
+ "keras-team__keras-19484",
+ "keras-team__keras-19775",
+ "keras-team__keras-19826",
+ "keras-team__keras-19838",
+ "keras-team__keras-19844",
+ "langchain-ai__langchain-19331",
+ "langchain-ai__langchain-20064",
+ "langchain-ai__langchain-3367",
+ "langchain-ai__langchain-4009",
+ "langchain-ai__langchain-4103",
+ "mrdoob__three.js-14836",
+ "mrdoob__three.js-18648",
+ "mrdoob__three.js-20991",
+ "mrdoob__three.js-24461",
+ "mui__material-ui-11451",
+ "mui__material-ui-11858",
+ "mui__material-ui-11987",
+ "mui__material-ui-12236",
+ "mui__material-ui-13534",
+ "mui__material-ui-13690",
+ "prettier__prettier-11000",
+ "prettier__prettier-11637",
+ "prettier__prettier-12930",
+ "prettier__prettier-14400",
+ "prettier__prettier-3436",
+ "prettier__prettier-3515",
+ "prettier__prettier-361",
+ "prettier__prettier-3723",
+ "prettier__prettier-459",
+ "prettier__prettier-666",
+ "prettier__prettier-8046",
+ "prettier__prettier-8777",
+ "serverless__serverless-1910",
+ "serverless__serverless-2014",
+ "serverless__serverless-2434",
+ "serverless__serverless-2576",
+ "serverless__serverless-2842",
+ "serverless__serverless-2945",
+ "serverless__serverless-2952",
+ "serverless__serverless-4192",
+ "serverless__serverless-5571",
+ "serverless__serverless-5640",
+ "serverless__serverless-5775",
+ "serverless__serverless-5842",
+ "sveltejs__svelte-1049",
+ "sveltejs__svelte-1095",
+ "sveltejs__svelte-1116",
+ "sveltejs__svelte-1137",
+ "sveltejs__svelte-1190",
+ "sveltejs__svelte-1227",
+ "sveltejs__svelte-1231",
+ "sveltejs__svelte-1310",
+ "sveltejs__svelte-1376",
+ "sveltejs__svelte-1923",
+ "sveltejs__svelte-1932",
+ "sveltejs__svelte-3702",
+ "sveltejs__svelte-3749",
+ "sveltejs__svelte-3949",
+ "sveltejs__svelte-4146",
+ "sveltejs__svelte-4288",
+ "tailwindlabs__tailwindcss-116",
+ "tailwindlabs__tailwindcss-550",
+ "tailwindlabs__tailwindcss-853",
+ "yt-dlp__yt-dlp-10390",
+ "yt-dlp__yt-dlp-4841",
+ "yt-dlp__yt-dlp-5195",
+ "yt-dlp__yt-dlp-5933",
+ "yt-dlp__yt-dlp-9862"
+ ],
+ "test": [
+ "apache__dubbo-4515",
+ "apache__dubbo-4567",
+ "apache__dubbo-5356",
+ "apache__rocketmq-7420",
+ "apache__rocketmq-7563",
+ "apache__rocketmq-7712",
+ "google__gson-2410",
+ "google__gson-2420",
+ "google__gson-2435",
+ "google__gson-2701",
+ "huggingface__transformers-16661",
+ "huggingface__transformers-22158",
+ "huggingface__transformers-23126",
+ "huggingface__transformers-26568",
+ "huggingface__transformers-26752",
+ "huggingface__transformers-28517",
+ "huggingface__transformers-30556",
+ "huggingface__transformers-30934",
+ "huggingface__transformers-6098",
+ "huggingface__transformers-7075",
+ "huggingface__transformers-7374",
+ "huggingface__transformers-7562",
+ "huggingface__transformers-8435",
+ "huggingface__transformers-8624",
+ "keras-team__keras-19937",
+ "microsoft__vscode-106767",
+ "microsoft__vscode-110094",
+ "microsoft__vscode-113837",
+ "microsoft__vscode-160342",
+ "microsoft__vscode-189223",
+ "mui__material-ui-14638",
+ "mui__material-ui-15359",
+ "mui__material-ui-18141",
+ "mui__material-ui-19511",
+ "mui__material-ui-24105",
+ "mui__material-ui-25874",
+ "mui__material-ui-26061",
+ "mui__material-ui-26807",
+ "mui__material-ui-34610",
+ "mui__material-ui-36971",
+ "prettier__prettier-4667",
+ "prettier__prettier-6604",
+ "prettier__prettier-9850",
+ "serverless__serverless-3095",
+ "serverless__serverless-5860",
+ "serverless__serverless-6534",
+ "serverless__serverless-7102",
+ "serverless__serverless-7374",
+ "serverless__serverless-7617",
+ "serverless__serverless-8159",
+ "sveltejs__svelte-3403",
+ "sveltejs__svelte-4332",
+ "sveltejs__svelte-510",
+ "sveltejs__svelte-5875",
+ "sveltejs__svelte-6759",
+ "trinodb__trino-1512",
+ "trinodb__trino-2707",
+ "trinodb__trino-2768",
+ "trinodb__trino-3603",
+ "trinodb__trino-4393"
+ ]
+}
\ No newline at end of file
diff --git a/ix-cli/scripts/ranking-eval/summarize.mjs b/ix-cli/scripts/ranking-eval/summarize.mjs
new file mode 100644
index 00000000..b123a41e
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/summarize.mjs
@@ -0,0 +1,45 @@
+// Tables from eval.mjs output: mean recall @5/@10/@20 per language and
+// overall, and per-instance wins/losses against a reference variant (a win is
+// a higher recall at that cutoff).
+//
+// node summarize.mjs results/dev-*.jsonl [--ref base] [--metric bm25|plan] [--md]
+import { readFileSync } from "node:fs";
+
+const argv = process.argv.slice(2);
+const opt = (name, dflt) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };
+const ref = opt("ref", "base");
+const metric = opt("metric", "bm25");
+const files = argv.filter((a, i) => !a.startsWith("--") && !argv[i - 1]?.startsWith("--"));
+const recs = files.flatMap((f) => readFileSync(f, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)))
+ .filter((r) => !r.skipped && r[metric]);
+const byVariant = new Map();
+for (const r of recs) {
+ if (!byVariant.has(r.variant)) byVariant.set(r.variant, new Map());
+ byVariant.get(r.variant).set(r.id, r);
+}
+const langs = [...new Set(recs.map((r) => r.lang))].sort();
+const mean = (xs) => xs.reduce((a, b) => a + b, 0) / (xs.length || 1);
+const f3 = (x) => x.toFixed(3);
+const refMap = byVariant.get(ref);
+console.log(`metric=${metric} ref=${ref} n=${refMap?.size}`);
+const head = ["variant", "@5", "@10", "@20", ...langs.map((l) => `${({ JavaScript: "JS", TypeScript: "TS", Python: "Py", Java: "Java" })[l] ?? l} @5/@10/@20`), "W/L @5", "W/L @10", "W/L @20"];
+console.log(head.join(" | "));
+for (const [v, m] of byVariant) {
+ const rs = [...m.values()];
+ const cells = [v, ...[0, 1, 2].map((k) => f3(mean(rs.map((r) => r[metric][k]))))];
+ for (const l of langs) {
+ const lr = rs.filter((r) => r.lang === l);
+ cells.push(`${[0, 1, 2].map((k) => mean(lr.map((r) => r[metric][k])).toFixed(2)).join("/")} (${lr.length})`);
+ }
+ for (const k of [0, 1, 2]) {
+ let w = 0, l = 0;
+ for (const r of rs) {
+ const b = refMap?.get(r.id);
+ if (!b) continue;
+ if (r[metric][k] > b[metric][k] + 1e-9) w++;
+ else if (r[metric][k] < b[metric][k] - 1e-9) l++;
+ }
+ cells.push(`${w}/${l}`);
+ }
+ console.log(cells.join(" | "));
+}
diff --git a/ix-cli/scripts/ranking-eval/variants.mjs b/ix-cli/scripts/ranking-eval/variants.mjs
new file mode 100644
index 00000000..7bae8352
--- /dev/null
+++ b/ix-cli/scripts/ranking-eval/variants.mjs
@@ -0,0 +1,162 @@
+// BM25 variants for the ranking study, one knob at a time. `base` reproduces
+// issue.ts's bm25Rank exactly (checked by eval.mjs --check); the rest change
+// one thing each, so a gain can be attributed.
+import { extractCandidates } from "./baseline-issue.mjs";
+
+const WORD_BASE = /[A-Za-z][A-Za-z0-9]*/g;
+const WORD_CODE = /[A-Za-z_][A-Za-z0-9_]*/g;
+const PART = /[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g;
+
+// Issue-prose words that say nothing about where the code is. English
+// function words, plus the words bug reports are written in.
+export const STOPWORDS = new Set((
+ "the and for are but not you all any can had her was one our out has him his how its may new now old see "
+ + "two way who did get let say she too use about above after again also been before being below between both "
+ + "could does doing down during each few from further have having here into just more most much must only other "
+ + "over own same should some such than that their them then there these they this those through under until very "
+ + "want what when where which while whom why will with would your yours "
+ + "bug bugs issue issues expected behavior behaviour actual reproduce reproduction steps version versions "
+ + "current currently describe description problem thanks thank please like think seems seem happen happens "
+ + "work works working instead because since using used example following sure able still even though"
+).split(/\s+/));
+
+/** Tokenizer: `base` is issue.ts's; `code` also keeps a multi-part identifier whole. */
+export function tokenize(text, o = {}) {
+ const minLen = o.minLen ?? 3;
+ const out = [];
+ if (!o.code) {
+ for (const word of text.match(WORD_BASE) ?? []) {
+ const parts = word.match(PART) ?? [word];
+ for (const part of parts) if (part.length >= minLen) out.push(part.toLowerCase());
+ }
+ return out;
+ }
+ for (const word of text.match(WORD_CODE) ?? []) {
+ const parts = word.match(PART) ?? [];
+ let n = 0;
+ for (const part of parts) {
+ n++;
+ if (part.length >= minLen) out.push(part.toLowerCase());
+ }
+ if (o.whole !== false && n >= 2) {
+ const whole = word.replace(/_/g, "").toLowerCase();
+ if (whole.length >= minLen) out.push(whole);
+ }
+ }
+ return out;
+}
+
+function queryWeights(query, o) {
+ const toks = tokenize(query, o).filter((t) => !(o.stop && STOPWORDS.has(t)));
+ const qtf = new Map();
+ for (const t of toks) qtf.set(t, (qtf.get(t) ?? 0) + 1);
+ const w = new Map();
+ for (const [t, f] of qtf) {
+ // BM25's query-term saturation (k3): a word the issue repeats counts more.
+ w.set(t, o.k3 ? ((o.k3 + 1) * f) / (o.k3 + f) : 1);
+ }
+ if (o.bigram) {
+ // "save model" in prose asks for `saveModel` / `save_model`, which the
+ // code tokenizer keeps whole as `savemodel`.
+ const seq = tokenize(query, { minLen: o.minLen });
+ for (let i = 0; i + 1 < seq.length; i++) {
+ if (STOPWORDS.has(seq[i]) || STOPWORDS.has(seq[i + 1])) continue;
+ const t = seq[i] + seq[i + 1];
+ if (!w.has(t)) w.set(t, o.bigram);
+ }
+ }
+ if (o.idW && o.idW !== 1) {
+ const { paths, identifiers } = extractCandidates(query);
+ const idTerms = new Set();
+ for (const id of [...identifiers, ...(o.idPaths ? paths : [])]) for (const t of tokenize(id, o)) idTerms.add(t);
+ for (const t of idTerms) if (w.has(t)) w.set(t, w.get(t) * o.idW);
+ }
+ return w;
+}
+
+/**
+ * Options:
+ * code, whole, minLen tokenizer
+ * stop drop STOPWORDS from the query
+ * k1, b BM25
+ * k3 query term frequency saturation (0: off, as base)
+ * idW, idPaths weight of query terms from identifiers extractCandidates finds
+ * pathW, pathB, pathField BM25F path field: weight, length norm, "full" | "base" (0: path is body text, as base)
+ * maxBytes skip files larger than this (base: 200KB)
+ * lenCap cap a doc's length at lenCap * avg in normalisation
+ * penalty (path) => multiplier on the final score
+ * exclude RegExp: paths left out of the corpus, as isSourcePath would
+ */
+export function rank(repo, files, query, o = {}) {
+ const k1 = o.k1 ?? 1.2;
+ const b = o.b ?? 0.75;
+ const maxBytes = o.maxBytes ?? 200 * 1024;
+ const qw = queryWeights(query, o);
+ const terms = [...qw.keys()];
+ const wanted = new Set(terms);
+ const docs = [];
+ for (const path of files) {
+ if (o.exclude?.test(path)) continue;
+ const text = repo.read(path);
+ if (text === undefined || Buffer.byteLength(text, "utf8") > maxBytes) continue;
+ const tf = new Map();
+ let length;
+ let ptf;
+ let plen = 0;
+ if (!o.pathW) {
+ const tokens = tokenize(`${path.replace(/\//g, " ")} ${text}`, o);
+ for (const t of tokens) if (wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);
+ length = tokens.length;
+ } else {
+ const tokens = tokenize(text, o);
+ for (const t of tokens) if (wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);
+ length = tokens.length;
+ const ptext = o.pathField === "base" ? path.slice(path.lastIndexOf("/") + 1) : path.replace(/\//g, " ");
+ const ptoks = tokenize(ptext.replace(/\.[A-Za-z]+$/, ""), o);
+ ptf = new Map();
+ for (const t of ptoks) if (wanted.has(t)) ptf.set(t, (ptf.get(t) ?? 0) + 1);
+ plen = ptoks.length;
+ }
+ docs.push({ path, tf, ptf, length, plen });
+ }
+ if (docs.length === 0) return [];
+ const n = docs.length;
+ let total = 0;
+ let ptotal = 0;
+ const df = new Map();
+ for (const d of docs) {
+ total += d.length;
+ ptotal += d.plen;
+ const seen = new Set(d.tf.keys());
+ if (d.ptf) for (const t of d.ptf.keys()) seen.add(t);
+ for (const t of seen) df.set(t, (df.get(t) ?? 0) + 1);
+ }
+ const avg = total / n || 1;
+ const pavg = ptotal / n || 1;
+ const pathB = o.pathB ?? 0.5;
+ const out = [];
+ for (const d of docs) {
+ const len = o.lenCap ? Math.min(d.length, o.lenCap * avg) : d.length;
+ const norm = 1 - b + (b * len) / avg;
+ const pnorm = 1 - pathB + (pathB * d.plen) / pavg;
+ let score = 0;
+ for (const t of terms) {
+ const f = d.tf.get(t) ?? 0;
+ const pf = d.ptf?.get(t) ?? 0;
+ if (!f && !pf) continue;
+ const dd = df.get(t);
+ const idf = Math.log(1 + (n - dd + 0.5) / (dd + 0.5));
+ let s;
+ if (!o.pathW) {
+ s = (f * (k1 + 1)) / (f + k1 * norm);
+ } else {
+ const tt = f / norm + (o.pathW * pf) / pnorm; // BM25F: fields combined before saturation
+ s = (tt * (k1 + 1)) / (tt + k1);
+ }
+ score += qw.get(t) * idf * s;
+ }
+ if (o.penalty) score *= o.penalty(d.path);
+ if (score > 0) out.push({ path: d.path, score });
+ }
+ return out.sort((a, c) => c.score - a.score || (a.path < c.path ? -1 : a.path > c.path ? 1 : 0));
+}