Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
ranking-eval: the offline BM25 study for --from-issue, summaries only
KageBinary023735917e1e
diff --git a/.gitignore b/.gitignoreindex 6e82a0d4..46dcf306 100644--- a/.gitignore+++ b/.gitignore@@ -46,4 +46,4 @@ ix-cli/.watch-child-runtime-*!.env.example# SBT boot artifacts-.sbt-boot/\ No newline at end of file+.sbt-boot/ix-cli/scripts/ranking-eval/results/*.jsonldiff --git a/ix-cli/scripts/ranking-eval/baseline-issue.mjs b/ix-cli/scripts/ranking-eval/baseline-issue.mjsnew file mode 100644index 00000000..3d3228c6--- /dev/null+++ b/ix-cli/scripts/ranking-eval/baseline-issue.mjs@@ -0,0 +1,383 @@+// Copyright 2026 Ix Infrastructure Inc.+// The compiled issue.ts of origin/main 198cebd (unchanged at abf39c8), before this branch,+// frozen so the baseline can be scored after issue.ts changes. tsc output, imports repointed.+import { readFileSync } from "node:fs";+import { posix } from "node:path";+import { isTestPath } from "../../dist/cli/explain/related-files.js";+import { resolveToken } from "../../dist/cli/explain/text-references.js";+/**+ * From an issue's text to the files its fix starts in, for+ * `ix context --from-issue`.+ *+ * Measured on SWE-PolyBench (28 fresh instances, not used while building+ * this): resolving the code names an issue mentions to their definitions put+ * a changed file in the top five 0.600 of the time, against 0.485 for BM25+ * over the repository. But a graph-expanded bundle, ranked by graph distance,+ * lost to BM25 at equal size (0.57 against 0.67). So the ranking here is+ * lexical first: the resolved starting files lead, BM25 against the issue+ * orders the rest, and graph closeness to a starting point only breaks ties+ * and nudges.+ *+ * Starting points, most specific first:+ *+ * 1. file paths the issue names, resolved against the tracked files;+ * 2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`+ * -- from backticks or running text. A multi-part name is almost always a+ * code name;+ * 3. single backticked words of four letters or more. `save` was the right+ * start for a keras fix; `flex`, `code` and `version` were not, which is+ * why a single word counts only in backticks, and only after the rest.+ *+ * A name resolves only to an exact-name definition in source code. The first+ * pilot run started from a CSS output file, a changelog, test fixtures, and an+ * import: `borderStylesReset` resolved to the file that imported it, because+ * the graph's `module` entity for an import is located in the importing file.+ */+/** Starting points a bundle is built from. */+export const MAX_STARTS = 3;+/** Candidate names looked up, in order. */+export const MAX_CANDIDATES = 25;+/** Files larger than this are not scored: generated, bundled or data. */+export const MAX_BM25_BYTES = 200 * 1024;+/** Ranked files carried in the JSON bundle. */+export const MAX_RANKED_FILES = 20;+/**+ * How much being next to a starting point is worth, as a fraction of a file's+ * BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at+ * equal size, so closeness may reorder files the text scores alike and may+ * not overturn a clear lexical lead.+ */+export const CLOSENESS_BOOST = 0.1;+/** Extensions of files a fix edits. */+const CODE_EXTENSIONS = [+ "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",+ "kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",+];+const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);+/**+ * Not code a fix edits: tests, test data, samples, docs, vendored or built+ * output. The pilot's list, plus what `isTestPath` already knows.+ */+const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"+ + "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased)/"+ + "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");+/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */+export function isSourcePath(path) {+ return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);+}+// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,+// and the absolute path of a stack trace frame or the tail of a repository URL,+// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which+// `resolveIssuePath` cuts down to the tracked path they end in.+//+// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then+// only begin where a run of path characters begins, so a long unbroken run --+// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.+const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");+const BACKTICK = /`([^`\n]{2,80})`/g;+const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;+// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.+const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";+const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");+const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);+/** The paths and code names an issue mentions, most specific first. */+export function extractCandidates(text) {+ const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));+ // A path's own parts are not names in the issue: `saving_api` in+ // `saving/saving_api.py` is the path again, not a function.+ const prose = text.replace(PATH, " ");+ const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));+ const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));+ const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);+ const seen = new Set(paths);+ const identifiers = [];+ for (const token of [...multi, ...single]) {+ if (seen.has(token))+ continue;+ seen.add(token);+ identifiers.push(token);+ }+ return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };+}+/** Kinds a definition has, most navigable first. Anything else ranks after. */+const DEFINITION_KINDS = [+ "class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",+ "constant", "variable", "property", "field",+];+/** Never a definition: an import (located in the importer), a chunk, a file. */+const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);+function kindRank(kind) {+ const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());+ return i === -1 ? DEFINITION_KINDS.length : i;+}+/**+ * Up to `max` starting points, first found wins, one per file; and the+ * candidates that resolved to nothing. Candidates after the last start are+ * not looked up, and not reported as unresolved.+ */+export async function pickStartingPoints(text, deps, max = MAX_STARTS) {+ const { paths, identifiers } = extractCandidates(text);+ const starts = [];+ const unresolved = [];+ const taken = (path) => starts.some((s) => s.path === path);+ const tracked = new Set(deps.files);+ const byBasename = new Map();+ for (const file of deps.files) {+ const base = posix.basename(file);+ byBasename.set(base, [...(byBasename.get(base) ?? []), file]);+ }+ for (const token of paths) {+ if (starts.length >= max)+ return { starts, unresolved };+ const path = resolveIssuePath(token, tracked, byBasename);+ if (!path || !isSourcePath(path)) {+ unresolved.push(token);+ continue;+ }+ if (!taken(path))+ starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });+ }+ const score = deps.fileScore ?? (() => 0);+ for (const token of identifiers) {+ if (starts.length >= max)+ break;+ // Not caught: a backend that cannot answer is an error to report, not an+ // issue that names nothing.+ const hits = (await deps.search(token))+ .filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))+ .sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));+ if (hits.length === 0) {+ unresolved.push(token);+ continue;+ }+ const best = hits[0];+ if (taken(best.path))+ continue;+ starts.push({+ token, id: best.id, name: best.name, kind: best.kind, path: best.path,+ ...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),+ ...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),+ via: "identifier in issue",+ });+ }+ return { starts, unresolved };+}+/**+ * A path as an issue writes it, to the tracked file it names. First as+ * `resolveToken` reads a path in code; then, for an absolute path from a+ * stack trace or the tail of a repository URL, the longest of its suffixes+ * that is a tracked path: `/home/u/proj/src/a.py` and+ * `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.+ */+function resolveIssuePath(token, tracked, byBasename) {+ const direct = resolveToken(token, "", tracked, byBasename);+ if (direct)+ return direct;+ const parts = token.split("/");+ for (let i = 1; i < parts.length; i++) {+ const suffix = parts.slice(i).join("/");+ if (tracked.has(suffix))+ return suffix;+ }+ return undefined;+}+/**+ * Words for BM25: identifiers split at camelCase and underscores, lowercased,+ * parts of two letters or fewer dropped. `listByKind` is `list kind`, so an+ * issue that says "list by kind" matches the code that spells it as one word.+ */+export function bm25Tokens(text) {+ const out = [];+ for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {+ const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];+ for (const part of parts)+ if (part.length > 2)+ out.push(part.toLowerCase());+ }+ return out;+}+/**+ * Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files+ * that share no word with it left out. A file's path is part of its text, so+ * `auth/login.ts` matches an issue about logging in before it is opened.+ */+export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {+ // Only the query's words are counted: BM25 needs no other term frequency,+ // and a map of every word of every file is most of the memory and a third+ // of the time on a large repository. A document's length is still every word.+ const terms = [...new Set(bm25Tokens(query))];+ const wanted = new Set(terms);+ const docs = new Map();+ for (const path of files) {+ const text = repo.read(path);+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)+ continue;+ const tf = new Map();+ const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);+ for (const t of tokens)+ if (wanted.has(t))+ tf.set(t, (tf.get(t) ?? 0) + 1);+ docs.set(path, { tf, length: tokens.length });+ }+ if (docs.size === 0)+ return [];+ const n = docs.size;+ let total = 0;+ const df = new Map();+ for (const doc of docs.values()) {+ total += doc.length;+ for (const t of doc.tf.keys())+ df.set(t, (df.get(t) ?? 0) + 1);+ }+ const avg = total / n || 1;+ const out = [];+ for (const [path, doc] of docs) {+ let score = 0;+ for (const t of terms) {+ const f = doc.tf.get(t);+ if (!f)+ continue;+ const d = df.get(t);+ const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));+ score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));+ }+ if (score > 0)+ out.push({ path, score });+ }+ return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));+}+/**+ * Starting points and the lexical ranking for one issue. Without a git+ * checkout there is no file list: paths do not resolve, BM25 is empty, and+ * only names can start the bundle.+ */+export async function planIssue(text, deps) {+ const files = deps.repo?.files() ?? [];+ const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];+ const scores = new Map(bm25.map((h) => [h.path, h.score]));+ const { starts, unresolved } = await pickStartingPoints(text, {+ files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,+ });+ if (starts.length > 0 || bm25.length === 0)+ return { starts, unresolved, fallback: false, bm25 };+ const top = bm25[0].path;+ return {+ starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],+ unresolved,+ fallback: true,+ bm25,+ };+}+/** BM25 files tried, best first, for one the graph has a node for. */+export const CENTRE_FALLBACK_TRIES = 10;+/**+ * The starting point a bundle is centred on: the first start the graph has a+ * node for, else the best BM25 file that has one.+ *+ * A start can have no node: the graph does not index every tracked file.+ * On SWE-PolyBench two svelte issues named no code, BM25's top file was a+ * `.svelte` component, and the command failed with no bundle at all while+ * the fifth-ranked file was a perfectly good `.js` module. Walking BM25's+ * ranking finds that one; the files the walk skipped still rank as usual.+ * `walked` says the centre came from this walk, which is a fallback too.+ */+export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {+ const located = await Promise.all(starts.map(async (s) => {+ const id = s.id ?? (await findNode(s.path));+ return id ? { ...s, id } : s;+ }));+ const found = located.find((s) => s.id);+ if (found)+ return { starts: located, centre: found, walked: false };+ const tried = new Set(located.map((s) => s.path));+ for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {+ const id = await findNode(hit.path);+ if (!id)+ continue;+ const centre = {+ token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",+ };+ return { starts: [...located, centre], centre, walked: true };+ }+ return { starts: located, walked: false };+}+/**+ * The files to read, best first: the starting files in the order they were+ * found, then every other source file by+ *+ * bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))+ *+ * where closeness is 1 for a file one hop from a starting point, 0.5 for one+ * of its ranked related files, 0 otherwise. Ties go to the closer file, then+ * the path. A graph neighbour the issue's text does not match at all still+ * ranks, after every file it does match.+ */+export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {+ const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));+ const out = [];+ const startPaths = new Set();+ for (const s of input.starts) {+ if (startPaths.has(s.path))+ continue;+ startPaths.add(s.path);+ out.push({+ path: s.path,+ score: round(lexical.get(s.path) ?? 0),+ reason: `starting point (${s.via}: ${s.token})`,+ });+ }+ const pool = new Set([...lexical.keys(), ...input.near.keys()]);+ const rest = [...pool]+ .filter((path) => !startPaths.has(path) && isSourcePath(path))+ .map((path) => {+ const bm25 = lexical.get(path) ?? 0;+ const near = input.near.get(path);+ const closeness = near?.weight ?? 0;+ const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]+ .filter(Boolean).join("; ");+ return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };+ })+ .sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));+ for (const r of rest)+ out.push({ path: r.path, score: round(r.score), reason: r.reason });+ return out.slice(0, limit);+}+/**+ * The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the+ * stream can be replaced under test.+ */+export async function readIssueText(arg, stdin = process.stdin) {+ if (arg !== "-")+ return decodeIssue(readFileSync(arg));+ const chunks = [];+ for await (const chunk of stdin) {+ chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));+ }+ return decodeIssue(Buffer.concat(chunks));+}+/**+ * UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`+ * writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >+ * issue.md` there produces a file that, read as UTF-8, is every letter+ * followed by a NUL: no name, no path, and no BM25 word in it.+ */+function decodeIssue(bytes) {+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {+ return bytes.subarray(2).toString("utf16le");+ }+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {+ const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));+ return swapped.swap16().toString("utf16le");+ }+ const text = bytes.toString("utf8");+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;+}+function unique(items) {+ return [...new Set(items)];+}+function round(n) {+ return Math.round(n * 1000) / 1000;+}+function cmp(a, b) {+ return a < b ? -1 : a > b ? 1 : 0;+}diff --git a/ix-cli/scripts/ranking-eval/configs.mjs b/ix-cli/scripts/ranking-eval/configs.mjsnew file mode 100644index 00000000..f73f4a6b--- /dev/null+++ b/ix-cli/scripts/ranking-eval/configs.mjs@@ -0,0 +1,99 @@+// Named variants. Round 1 changes one thing from `base` each; later rounds+// combine what won on dev.+const DOCS_SITE = /(^|\/)(site|website|guides|demos?|playground|benchmarks?)\//;+const DTS = /\.d\.ts$/;++export const VARIANTS = {+ base: {},+ // tokenizer+ code: { code: true },+ stop: { stop: true },+ min4: { minLen: 4 },+ // query weighting+ qtf: { k3: 1.5 },+ qtf8: { k3: 8 },+ id2: { idW: 2 },+ id3: { idW: 3 },+ id5: { idW: 5 },+ id3p: { idW: 3, idPaths: true },+ // BM25F path field+ path1: { pathW: 1 },+ path2: { pathW: 2 },+ path4: { pathW: 4 },+ path2base: { pathW: 2, pathField: "base" },+ path4base: { pathW: 4, pathField: "base" },+ // length normalisation+ k1_09: { k1: 0.9 },+ k1_15: { k1: 1.5 },+ k1_20: { k1: 2.0 },+ b03: { b: 0.3 },+ b05: { b: 0.5 },+ b09: { b: 0.9 },+ cap3: { lenCap: 3 },+ max100: { maxBytes: 100 * 1024 },+ // file kinds+ nosite: { penalty: (p) => (DOCS_SITE.test(p) ? 0.5 : 1) },+ nodts: { penalty: (p) => (DTS.test(p) ? 0.5 : 1) },+};++const CS = { code: true, stop: true };+Object.assign(VARIANTS, {+ cs: CS,+ cs_q8: { ...CS, k3: 8 },+ cs_id2: { ...CS, idW: 2 },+ cs_id3: { ...CS, idW: 3 },+ cs_q8_id2: { ...CS, k3: 8, idW: 2 },+ cs_site: { ...CS, penalty: VARIANTS.nosite.penalty },+ cs_k15: { ...CS, k1: 1.5 },+ cs_k20: { ...CS, k1: 2.0 },+ cs_big05: { ...CS, bigram: 0.5 },+ cs_big1: { ...CS, bigram: 1 },+ cs_path2base: { ...CS, pathW: 2, pathField: "base" },+ max256: { maxBytes: 256 * 1024 },+ code_big1: { code: true, bigram: 1 },+});++// Round 3: on top of code tokens + stopwords + doc-site penalty.+const CSS = { ...CS, penalty: VARIANTS.nosite.penalty };+const only = (re) => (p) => (re.test(p) ? 0 : 1);+Object.assign(VARIANTS, {+ cs_site0: { ...CS, penalty: only(DOCS_SITE) },+ cs_siteonly: { ...CS, penalty: only(/(^|\/)(site|website)\//) },+ css_id2: { ...CSS, idW: 2 },+ css_id3: { ...CSS, idW: 3 },+ css_q8: { ...CSS, k3: 8 },+ css_q2: { ...CSS, k3: 2 },+ css_k15: { ...CSS, k1: 1.5 },+ css_k10: { ...CSS, k1: 1.0 },+ css_b06: { ...CSS, b: 0.6 },+ css_b09: { ...CSS, b: 0.9 },+ css_big05: { ...CSS, bigram: 0.5 },+});++// The chosen variant, as issue.ts implements it: code tokens, prose+// stopwords, identifier terms x2, and docs-site directories out of the corpus.+Object.assign(VARIANTS, {+ final: { ...CS, idW: 2, exclude: DOCS_SITE },+});++// Ablations of `final`: each leaves one of its four parts out.+Object.assign(VARIANTS, {+ final_nocode: { stop: true, idW: 2, exclude: DOCS_SITE },+ final_nostop: { code: true, idW: 2, exclude: DOCS_SITE },+ final_noid: { ...CS, exclude: DOCS_SITE },+ final_nosite: { ...CS, idW: 2 },+});++// v2, after the coordinator's check against all 382 issues' gold: guides/,+// demo(s)/ and benchmark/ hold files real fixes change, so an exclusion rule+// must hold across the whole dataset's gold, not just dev's. isSourcePath now+// comes from fix/issue-start-picking, whose list was checked against all 932+// gold files; `final2` is `final` with that list in place of DOCS_SITE.+const PICKS_NOISE = /(^|\/)(site|website|docs-site|playground|tests?[_-]config|benchmarks|\.?storybook|e2e|cypress|integration-tests)\//;+Object.assign(VARIANTS, {+ picks_base: { exclude: PICKS_NOISE },+ final2: { ...CS, idW: 2, exclude: PICKS_NOISE },+ final2_nocode: { stop: true, idW: 2, exclude: PICKS_NOISE },+ final2_nostop: { code: true, idW: 2, exclude: PICKS_NOISE },+ final2_noid: { ...CS, exclude: PICKS_NOISE },+});diff --git a/ix-cli/scripts/ranking-eval/corpus.mjs b/ix-cli/scripts/ranking-eval/corpus.mjsnew file mode 100644index 00000000..2f0e4664--- /dev/null+++ b/ix-cli/scripts/ranking-eval/corpus.mjs@@ -0,0 +1,58 @@+// Read a repository snapshot straight from ix-bench's bare clones: no+// checkout, no backend. `git ls-tree` lists the tracked files at the base+// commit; `git cat-file --batch` reads the source ones in one process.+import { execFileSync, spawnSync } from "node:child_process";+import { homedir } from "node:os";+import { join } from "node:path";++import { isSourcePath } from "./baseline-issue.mjs";++export const REPOS = join(homedir(), ".cache", "ix-bench", "external", "repos");+/** What `gitRepoAccess` reads at most (text-references MAX_SCAN_BYTES). */+const MAX_READ_BYTES = 256 * 1024;++export function bareRepo(repo) {+ return join(REPOS, `${repo.replace("/", "__")}.git`);+}++/** A RepoAccess over `commit`: every tracked file listed, source files readable. */+export function loadCorpus(repo, commit) {+ const cwd = bareRepo(repo);+ const listing = execFileSync("git", ["ls-tree", "-r", "-z", "--long", commit], {+ cwd, encoding: "utf8", maxBuffer: 256 * 1024 * 1024,+ }).split("\0").filter(Boolean);+ const files = [];+ const wanted = [];+ for (const row of listing) {+ const tab = row.indexOf("\t");+ const [, type, oid, size] = row.slice(0, tab).trim().split(/\s+/);+ const path = row.slice(tab + 1);+ if (type !== "blob") continue;+ files.push(path);+ if (isSourcePath(path) && Number(size) <= MAX_READ_BYTES) wanted.push({ path, oid });+ }+ const texts = new Map();+ if (wanted.length) {+ const out = spawnSync("git", ["cat-file", "--batch"], {+ cwd, input: wanted.map((w) => w.oid).join("\n") + "\n", maxBuffer: 2 * 1024 * 1024 * 1024,+ }).stdout;+ let pos = 0;+ for (const w of wanted) {+ const nl = out.indexOf(10, pos);+ const header = out.toString("utf8", pos, nl).split(" ");+ const len = Number(header[2]);+ texts.set(w.path, out.toString("utf8", nl + 1, nl + 1 + len));+ pos = nl + 1 + len + 1;+ }+ }+ return {+ files: () => files,+ read: (path) => texts.get(path),+ grep: () => [],+ texts,+ };+}++export function goldFiles(patch) {+ return [...new Set([...patch.matchAll(/^diff --git a\/(\S+) b\//gm)].map((m) => m[1]))];+}diff --git a/ix-cli/scripts/ranking-eval/defs.mjs b/ix-cli/scripts/ranking-eval/defs.mjsnew file mode 100644index 00000000..0e2592af--- /dev/null+++ b/ix-cli/scripts/ranking-eval/defs.mjs@@ -0,0 +1,37 @@+// A stand-in for `ix search --name-only` with no backend: the definitions of+// a name, found by regex over the snapshot's source files. Only good enough to+// measure the plan's ranking (starting files, then BM25) offline; the graph's+// own resolver is what the CLI uses.+const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");++const LEADING_NOT_DEF = /\b(return|new|if|while|for|switch|catch|else|await|throw|yield|typeof|case)\b/;++export function defSearch(repo, files) {+ return async (name) => {+ const n = esc(name);+ const pats = [+ [new RegExp(`\\b(class|interface|enum|struct|trait|record|object)\\s+${n}\\b`), (m) => (m[1] === "record" ? "class" : m[1])],+ [new RegExp(`\\btype\\s+${n}\\b\\s*[=<]`), () => "type"],+ [new RegExp(`\\b(?:function\\s*\\*?|def|fun|fn|func)\\s+${n}\\b`), () => "function"],+ [new RegExp(`\\b(?:const|let|var)\\s+${n}\\s*=\\s*(?:async\\s*)?(?:function\\b|\\([^)]*\\)\\s*=>|[A-Za-z_$][\\w$]*\\s*=>)`), () => "function"],+ [new RegExp(`^[ \\t]*((?:[\\w<>\\[\\],.?@]+[ \\t]+)*)${n}[ \\t]*(?:<[^>\\n]*>)?\\([^\\n;]*\\)[^\\n;]*\\{[ \\t]*$`, "m"),+ (m) => (LEADING_NOT_DEF.test(m[1]) ? undefined : "method")],+ [new RegExp(`\\b(?:const|let|var|val|final\\s+[\\w<>\\[\\]]+|static\\s+[\\w<>\\[\\]]+)\\s+${n}\\b`), () => (/^[A-Z0-9_]+$/.test(name) ? "constant" : "variable")],+ [new RegExp(`^${n}\\s*=`, "m"), () => (/^[A-Z0-9_]+$/.test(name) ? "constant" : "variable")],+ ];+ const hits = [];+ for (const path of files) {+ const text = repo.read(path);+ if (!text || !text.includes(name)) continue;+ for (const [re, kindOf] of pats) {+ const m = text.match(re);+ const kind = m && kindOf(m);+ if (kind) {+ hits.push({ id: `${path}#${name}`, name, kind, path });+ break;+ }+ }+ }+ return hits;+ };+}diff --git a/ix-cli/scripts/ranking-eval/eval.mjs b/ix-cli/scripts/ranking-eval/eval.mjsnew file mode 100644index 00000000..c7f1a293--- /dev/null+++ b/ix-cli/scripts/ranking-eval/eval.mjs@@ -0,0 +1,98 @@+// Score BM25 variants on the dev (or, once, the test) split.+//+// node eval.mjs --set dev --variants base,code --out results/dev.jsonl [--plan] [--check]+//+// For each instance: read the snapshot at base_commit from ix-bench's bare+// clone, rank the tracked source files against the problem statement with+// each variant, and record file recall @5/@10/@20 against the gold patch's+// source files that exist at the base commit. --plan also scores the plan's+// order (starting files from pickStartingPoints with a regex definition+// search standing in for the graph, then BM25) -- rankIssueFiles with no+// graph closeness. --check asserts `base` equals the baseline bm25Rank.+// The variant `impl` is the built dist/ (the current issue.ts): its own+// isSourcePath, bm25Rank, pickStartingPoints and rankIssueFiles. The variant+// `picks` is the same for fix/issue-start-picking, the branch this is on.+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";+import { dirname } from "node:path";+import { fileURLToPath } from "node:url";++import * as dist from "../../dist/cli/explain/issue.js";+// Gold, the file list and the `base` plan use issue.ts as it was before this+// branch, so the baseline and the scoring stay fixed while issue.ts changes.+import { bm25Rank, isSourcePath, pickStartingPoints, rankIssueFiles } from "./baseline-issue.mjs";+import * as picks from "./picks-issue.mjs";+import { goldFiles, loadCorpus } from "./corpus.mjs";+import { defSearch } from "./defs.mjs";+import { VARIANTS } from "./configs.mjs";+import { rank } from "./variants.mjs";++const HERE = dirname(fileURLToPath(import.meta.url));+const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {+ if (a.startsWith("--")) acc.push([a.slice(2), all[i + 1] && !all[i + 1].startsWith("--") ? all[i + 1] : true]);+ return acc;+}, []));+const set = args.set ?? "dev";+if (set === "test" && args.final !== "yes-once") {+ console.error("The test split is scored once, for the final variant: pass --final yes-once.");+ process.exit(2);+}+const split = JSON.parse(readFileSync(`${HERE}/split.json`, "utf8"));+const ids = new Set(split[set]);+const only = args.only ? new Set(String(args.only).split(",")) : undefined;+const rows = readFileSync("/home/ihock/src/ix-bench/results/external/polybench_verified.jsonl", "utf8")+ .split("\n").filter(Boolean).map((l) => JSON.parse(l))+ .filter((r) => ids.has(r.instance_id) && (!only || only.has(r.instance_id)));+const MODULES = { impl: dist, picks };+const variantNames = String(args.variants ?? "base").split(",");+for (const v of variantNames) if (!VARIANTS[v] && !MODULES[v]) throw new Error(`unknown variant ${v}`);+const out = args.out ?? `${HERE}/results/${set}.jsonl`;+mkdirSync(dirname(out), { recursive: true });++const recall = (pred, gold) => gold.filter((g) => pred.includes(g)).length / gold.length;+const at = (list, gold) => [5, 10, 20].map((k) => recall(list.slice(0, k), gold));++const lines = [];+let done = 0;+for (const row of rows) {+ const t0 = Date.now();+ const repo = loadCorpus(row.repo, row.base_commit);+ const all = repo.files();+ const tracked = new Set(all);+ const source = all.filter(isSourcePath);+ const gold = goldFiles(row.patch).filter((p) => isSourcePath(p) && tracked.has(p));+ const loadMs = Date.now() - t0;+ if (gold.length === 0) {+ lines.push({ id: row.instance_id, lang: row.language, repo: row.repo, skipped: "no source gold at base" });+ continue;+ }+ const query = row.problem_statement;+ if (args.check) {+ const a = bm25Rank(repo, source, query).map((h) => h.path).slice(0, 50);+ const b = rank(repo, source, query, VARIANTS.base).map((h) => h.path).slice(0, 50);+ if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error(`base != bm25Rank on ${row.instance_id}`);+ }+ const search = args.plan ? defSearch(repo, source) : undefined;+ for (const v of variantNames) {+ const t1 = Date.now();+ const mod = MODULES[v];+ const hits = mod ? mod.bm25Rank(repo, all.filter(mod.isSourcePath), query) : rank(repo, source, query, VARIANTS[v]);+ const ms = Date.now() - t1;+ const ranked = hits.map((h) => h.path);+ const rec = { id: row.instance_id, lang: row.language, repo: row.repo, variant: v, gold, ms, loadMs,+ nfiles: source.length, bm25: at(ranked, gold), top: ranked.slice(0, 20) };+ if (search) {+ // Starting points depend on BM25 only through fileScore's tie-break.+ const scores = new Map(hits.map((h) => [h.path, h.score]));+ const lib = mod ?? { pickStartingPoints, rankIssueFiles };+ const { starts } = await lib.pickStartingPoints(query, { files: all, search, fileScore: (p) => scores.get(p) ?? 0 });+ const planned = lib.rankIssueFiles({ starts, bm25: hits, near: new Map() }).map((r) => r.path);+ rec.plan = at(planned, gold);+ rec.starts = starts.map((s) => s.path);+ }+ lines.push(rec);+ }+ done++;+ process.stderr.write(`${done}/${rows.length} ${row.instance_id} files=${source.length} load=${loadMs}ms\n`);+}+writeFileSync(out, lines.map((l) => JSON.stringify(l)).join("\n") + "\n");+console.error(`wrote ${out}`);diff --git a/ix-cli/scripts/ranking-eval/make_split.py b/ix-cli/scripts/ranking-eval/make_split.pynew file mode 100644index 00000000..6c9b97bf--- /dev/null+++ b/ix-cli/scripts/ranking-eval/make_split.py@@ -0,0 +1,41 @@+"""Fix the dev/test split before any results are looked at.++DEV = every instance id in the earlier ix-bench runs (tuning allowed).+TEST = 60 instances from the rest, seed 20260930, stratified by language+ (15 per language), scored once at the end.+"""+import json, random, collections, pathlib+EXT = pathlib.Path("/home/ihock/src/ix-bench/results/external")+DEV_FILES = ["polybench_pilot.jsonl", "polybench_v2_first.jsonl", "polybench_v2_heldout.jsonl",+ "polybench_v3_first.jsonl", "polybench_v3_heldout.jsonl", "polybench_v3_fresh.jsonl",+ "from_issue_accept.jsonl"]+dev = set()+for f in DEV_FILES:+ for line in open(EXT / f):+ if line.strip():+ dev.add(json.loads(line)["instance_id"])+dev |= set(json.load(open(EXT / "e2e_instances.json")))+rows = [json.loads(l) for l in open(EXT / "polybench_verified.jsonl")]+ids = {r["instance_id"] for r in rows}+dev &= ids+rest = sorted((r for r in rows if r["instance_id"] not in dev), key=lambda r: r["instance_id"])+by_lang = collections.defaultdict(list)+for r in rest:+ by_lang[r["language"]].append(r["instance_id"])+rng = random.Random(20260930)+test = []+langs = sorted(by_lang)+per = 60 // len(langs)+for lang in langs:+ pool = by_lang[lang]+ test += rng.sample(pool, min(per, len(pool)))+# top up if a language was short+short = 60 - len(test)+if short:+ left = [i for l in langs for i in by_lang[l] if i not in test]+ test += rng.sample(left, short)+json.dump({"seed": 20260930, "dev": sorted(dev), "test": sorted(test)}, open("split.json", "w"), indent=1)+lang = {r["instance_id"]: r["language"] for r in rows}+print("dev", len(dev), collections.Counter(lang[i] for i in dev))+print("rest", len(rest), {l: len(v) for l, v in by_lang.items()})+print("test", len(test), collections.Counter(lang[i] for i in test))diff --git a/ix-cli/scripts/ranking-eval/picks-issue.mjs b/ix-cli/scripts/ranking-eval/picks-issue.mjsnew file mode 100644index 00000000..c22cd011--- /dev/null+++ b/ix-cli/scripts/ranking-eval/picks-issue.mjs@@ -0,0 +1,457 @@+// Copyright 2026 Ix Infrastructure Inc.+// issue.ts of fix/issue-start-picking (b62b524), the branch this one is stacked on, frozen as a+// second baseline: its wider isSourcePath and name order, the old BM25. Transpiled; imports repointed.+import { readFileSync } from "node:fs";+import { posix } from "node:path";+import { isTestPath } from "../../dist/cli/explain/related-files.js";+import { resolveToken } from "../../dist/cli/explain/text-references.js";+/**+ * From an issue's text to the files its fix starts in, for+ * `ix context --from-issue`.+ *+ * Measured on SWE-PolyBench (28 fresh instances, not used while building+ * this): resolving the code names an issue mentions to their definitions put+ * a changed file in the top five 0.600 of the time, against 0.485 for BM25+ * over the repository. But a graph-expanded bundle, ranked by graph distance,+ * lost to BM25 at equal size (0.57 against 0.67). So the ranking here is+ * lexical first: the resolved starting files lead, BM25 against the issue+ * orders the rest, and graph closeness to a starting point only breaks ties+ * and nudges.+ *+ * Starting points, most specific first:+ *+ * 1. file paths the issue names, resolved against the tracked files;+ * 2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`+ * -- from backticks or running text. A multi-part name is almost always a+ * code name;+ * 3. single backticked words of four letters or more. `save` was the right+ * start for a keras fix; `flex`, `code` and `version` were not, which is+ * why a single word counts only in backticks, and only after the rest.+ *+ * A name resolves only to an exact-name definition in source code. The first+ * pilot run started from a CSS output file, a changelog, test fixtures, and an+ * import: `borderStylesReset` resolved to the file that imported it, because+ * the graph's `module` entity for an import is located in the importing file.+ */+/** Starting points a bundle is built from. */+export const MAX_STARTS = 3;+/** Candidate names looked up, in order. */+export const MAX_CANDIDATES = 25;+/** Files larger than this are not scored: generated, bundled or data. */+export const MAX_BM25_BYTES = 200 * 1024;+/** Ranked files carried in the JSON bundle. */+export const MAX_RANKED_FILES = 20;+/**+ * How much being next to a starting point is worth, as a fraction of a file's+ * BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at+ * equal size, so closeness may reorder files the text scores alike and may+ * not overturn a clear lexical lead.+ */+export const CLOSENESS_BOOST = 0.1;+/** Extensions of files a fix edits. */+const CODE_EXTENSIONS = [+ "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",+ "kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",+];+const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);+/**+ * Not code a fix edits: tests, test data, samples, docs, vendored or built+ * output. The pilot's list, plus what `isTestPath` already knows.+ *+ * The second line is a project's website, playground, test configuration,+ * benchmarks, stories and end-to-end suites. Svelte's `site/` alone was 40+ * of the wrong files in Ix's top-10 rankings over two samples of issues; none+ * of these directories holds any of the 932 files the 382 SWE-PolyBench+ * Verified fixes change.+ */+const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"+ + "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased|"+ + "site|website|docs-site|playground|tests?[_-]config|benchmarks|\\.?storybook|e2e|cypress|integration-tests)/"+ + "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");+/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */+export function isSourcePath(path) {+ return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);+}+// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,+// and the absolute path of a stack trace frame or the tail of a repository URL,+// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which+// `resolveIssuePath` cuts down to the tracked path they end in.+//+// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then+// only begin where a run of path characters begins, so a long unbroken run --+// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.+const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");+const BACKTICK = /`([^`\n]{2,80})`/g;+const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;+// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.+const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";+const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");+const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);+/** The paths and code names an issue mentions, most specific first. */+export function extractCandidates(text) {+ const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));+ // A path's own parts are not names in the issue: `saving_api` in+ // `saving/saving_api.py` is the path again, not a function.+ const prose = text.replace(PATH, " ");+ const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));+ const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));+ const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);+ // Only MAX_STARTS names become starts, so order decides: a short common+ // word (`repeat`, `ignore`) mentioned first took a slot a specific one+ // (`Stylesheet`) mentioned later needed. Common words still resolve, last.+ const ordered = [+ ...multi,+ ...single.filter((t) => !isGenericIdentifier(t)),+ ...single.filter((t) => isGenericIdentifier(t)),+ ];+ const seen = new Set(paths);+ const identifiers = [];+ for (const token of ordered) {+ if (seen.has(token))+ continue;+ seen.add(token);+ identifiers.push(token);+ }+ return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };+}+/** Kinds a definition has, most navigable first. Anything else ranks after. */+const DEFINITION_KINDS = [+ "class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",+ "constant", "variable", "property", "field",+];+/** Never a definition: an import (located in the importer), a chunk, a file. */+const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);+function kindRank(kind) {+ const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());+ return i === -1 ? DEFINITION_KINDS.length : i;+}+/**+ * Up to `max` starting points, first found wins, one per file; and the+ * candidates that resolved to nothing. Candidates after the last start are+ * not looked up, and not reported as unresolved.+ */+export async function pickStartingPoints(text, deps, max = MAX_STARTS) {+ const { paths, identifiers } = extractCandidates(text);+ const starts = [];+ const unresolved = [];+ const taken = (path) => starts.some((s) => s.path === path);+ const tracked = new Set(deps.files);+ const byBasename = new Map();+ for (const file of deps.files) {+ const base = posix.basename(file);+ byBasename.set(base, [...(byBasename.get(base) ?? []), file]);+ }+ for (const token of paths) {+ if (starts.length >= max)+ return { starts, unresolved };+ const path = resolveIssuePath(token, tracked, byBasename);+ if (!path || !isSourcePath(path)) {+ unresolved.push(token);+ continue;+ }+ if (!taken(path))+ starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });+ }+ const score = deps.fileScore ?? (() => 0);+ for (const token of identifiers) {+ if (starts.length >= max)+ break;+ // Not caught: a backend that cannot answer is an error to report, not an+ // issue that names nothing.+ const hits = (await deps.search(token))+ .filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))+ .sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));+ if (hits.length === 0) {+ unresolved.push(token);+ continue;+ }+ const best = hits[0];+ if (taken(best.path))+ continue;+ starts.push({+ token, id: best.id, name: best.name, kind: best.kind, path: best.path,+ ...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),+ ...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),+ via: "identifier in issue",+ });+ }+ return { starts, unresolved };+}+/**+ * A path as an issue writes it, to the tracked file it names. First as+ * `resolveToken` reads a path in code; then, for an absolute path from a+ * stack trace or the tail of a repository URL, the longest of its suffixes+ * that is a tracked path: `/home/u/proj/src/a.py` and+ * `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.+ */+function resolveIssuePath(token, tracked, byBasename) {+ const direct = resolveToken(token, "", tracked, byBasename);+ if (direct)+ return direct;+ const parts = token.split("/");+ for (let i = 1; i < parts.length; i++) {+ const suffix = parts.slice(i).join("/");+ if (tracked.has(suffix))+ return suffix;+ }+ return undefined;+}+/**+ * Words for BM25: identifiers split at camelCase and underscores, lowercased,+ * parts of two letters or fewer dropped. `listByKind` is `list kind`, so an+ * issue that says "list by kind" matches the code that spells it as one word.+ */+export function bm25Tokens(text) {+ const out = [];+ for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {+ const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];+ for (const part of parts)+ if (part.length > 2)+ out.push(part.toLowerCase());+ }+ return out;+}+/**+ * Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files+ * that share no word with it left out. A file's path is part of its text, so+ * `auth/login.ts` matches an issue about logging in before it is opened.+ */+export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {+ // Only the query's words are counted: BM25 needs no other term frequency,+ // and a map of every word of every file is most of the memory and a third+ // of the time on a large repository. A document's length is still every word.+ const terms = [...new Set(bm25Tokens(query))];+ const wanted = new Set(terms);+ const docs = new Map();+ for (const path of files) {+ const text = repo.read(path);+ if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)+ continue;+ const tf = new Map();+ const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);+ for (const t of tokens)+ if (wanted.has(t))+ tf.set(t, (tf.get(t) ?? 0) + 1);+ docs.set(path, { tf, length: tokens.length });+ }+ if (docs.size === 0)+ return [];+ const n = docs.size;+ let total = 0;+ const df = new Map();+ for (const doc of docs.values()) {+ total += doc.length;+ for (const t of doc.tf.keys())+ df.set(t, (df.get(t) ?? 0) + 1);+ }+ const avg = total / n || 1;+ const out = [];+ for (const [path, doc] of docs) {+ let score = 0;+ for (const t of terms) {+ const f = doc.tf.get(t);+ if (!f)+ continue;+ const d = df.get(t);+ const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));+ score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));+ }+ if (score > 0)+ out.push({ path, score });+ }+ return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));+}+/**+ * Starting points and the lexical ranking for one issue. Without a git+ * checkout there is no file list: paths do not resolve, BM25 is empty, and+ * only names can start the bundle.+ */+export async function planIssue(text, deps) {+ const files = deps.repo?.files() ?? [];+ const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];+ const scores = new Map(bm25.map((h) => [h.path, h.score]));+ const { starts, unresolved } = await pickStartingPoints(text, {+ files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,+ });+ if (starts.length > 0 || bm25.length === 0)+ return { starts, unresolved, fallback: false, bm25 };+ const top = bm25[0].path;+ return {+ starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],+ unresolved,+ fallback: true,+ bm25,+ };+}+/**+ * Whether an identifier the issue names is too common to point anywhere: a+ * short all-lowercase word such as `debug`, `bind`, `config` or `render`+ * resolves to *a* definition, rarely the one the fix touches. Specific names+ * carry an inner capital (`toJsonTree`, `JsonWriter`), an underscore+ * (`get_openai_callback`), or length.+ *+ * Fixed before it was scored on held-out samples: on SWE-PolyBench issues the+ * starts it keeps were in the gold patch 20 times in 32, the ones it drops 2+ * in 7.+ */+export function isGenericIdentifier(token) {+ return !(/[a-z][A-Z]/.test(token) || /^[A-Z][a-z]+[A-Z]/.test(token) || token.includes("_") || token.length >= 8);+}+/**+ * A plan is confident when the issue named a path, or a specific identifier,+ * that resolved. A BM25 fallback never is: across four samples of+ * SWE-PolyBench issues its start was in the gold patch once in 55.+ */+export function issueConfidence(plan) {+ if (plan.fallback) {+ return { confident: false, starts: [], reason: "no code name in the issue resolved to a definition" };+ }+ const starts = plan.starts.filter((s) => s.via === "path in issue"+ || (s.via === "identifier in issue" && !isGenericIdentifier(s.token)));+ if (starts.length === 0) {+ const names = plan.starts.map((s) => s.token);+ return {+ confident: false,+ starts,+ reason: names.length > 0+ ? `only common words resolved (${names.join(", ")}), which name many definitions`+ : "nothing in the issue resolved to a definition",+ };+ }+ return { confident: true, starts };+}+/** Files after the starting points in a lean view. */+export const LEAN_RANKED = 4;+/**+ * The lean view of a plan. A full bundle costs about 1.5k tokens, and an+ * agent re-reads it on every turn; on SWE-PolyBench it added a median 14%+ * tokens even where its start was right, and saved no turns. What the agent+ * uses is the pointer, so that is all this keeps: the trusted starting points+ * and a few ranked files -- Ix's starts, then BM25, the ordering that beat+ * both alone on held-out issues. When nothing is trusted it says so, rather+ * than pointing the agent at a guess.+ */+export function leanIssueView(plan, ranked = LEAN_RANKED) {+ const c = issueConfidence(plan);+ if (!c.confident) {+ return { confidence: "low", reason: c.reason, startingPoints: [], alsoRanked: [], unresolved: plan.unresolved };+ }+ const startPaths = new Set(c.starts.map((s) => s.path));+ const alsoRanked = rankIssueFiles({ starts: [], bm25: plan.bm25, near: new Map() }, ranked + startPaths.size)+ .filter((f) => !startPaths.has(f.path))+ .slice(0, ranked);+ return { confidence: "high", startingPoints: c.starts, alsoRanked, unresolved: plan.unresolved };+}+/** BM25 files tried, best first, for one the graph has a node for. */+export const CENTRE_FALLBACK_TRIES = 10;+/**+ * The starting point a bundle is centred on: the first start the graph has a+ * node for, else the best BM25 file that has one.+ *+ * A start can have no node: the graph does not index every tracked file.+ * On SWE-PolyBench two svelte issues named no code, BM25's top file was a+ * `.svelte` component, and the command failed with no bundle at all while+ * the fifth-ranked file was a perfectly good `.js` module. Walking BM25's+ * ranking finds that one; the files the walk skipped still rank as usual.+ * `walked` says the centre came from this walk, which is a fallback too.+ */+export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {+ const located = await Promise.all(starts.map(async (s) => {+ const id = s.id ?? (await findNode(s.path));+ return id ? { ...s, id } : s;+ }));+ const found = located.find((s) => s.id);+ if (found)+ return { starts: located, centre: found, walked: false };+ const tried = new Set(located.map((s) => s.path));+ for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {+ const id = await findNode(hit.path);+ if (!id)+ continue;+ const centre = {+ token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",+ };+ return { starts: [...located, centre], centre, walked: true };+ }+ return { starts: located, walked: false };+}+/**+ * The files to read, best first: the starting files in the order they were+ * found, then every other source file by+ *+ * bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))+ *+ * where closeness is 1 for a file one hop from a starting point, 0.5 for one+ * of its ranked related files, 0 otherwise. Ties go to the closer file, then+ * the path. A graph neighbour the issue's text does not match at all still+ * ranks, after every file it does match.+ */+export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {+ const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));+ const out = [];+ const startPaths = new Set();+ for (const s of input.starts) {+ if (startPaths.has(s.path))+ continue;+ startPaths.add(s.path);+ out.push({+ path: s.path,+ score: round(lexical.get(s.path) ?? 0),+ reason: `starting point (${s.via}: ${s.token})`,+ });+ }+ const pool = new Set([...lexical.keys(), ...input.near.keys()]);+ const rest = [...pool]+ .filter((path) => !startPaths.has(path) && isSourcePath(path))+ .map((path) => {+ const bm25 = lexical.get(path) ?? 0;+ const near = input.near.get(path);+ const closeness = near?.weight ?? 0;+ const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]+ .filter(Boolean).join("; ");+ return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };+ })+ .sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));+ for (const r of rest)+ out.push({ path: r.path, score: round(r.score), reason: r.reason });+ return out.slice(0, limit);+}+/**+ * The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the+ * stream can be replaced under test.+ */+export async function readIssueText(arg, stdin = process.stdin) {+ if (arg !== "-")+ return decodeIssue(readFileSync(arg));+ const chunks = [];+ for await (const chunk of stdin) {+ chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));+ }+ return decodeIssue(Buffer.concat(chunks));+}+/**+ * UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`+ * writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >+ * issue.md` there produces a file that, read as UTF-8, is every letter+ * followed by a NUL: no name, no path, and no BM25 word in it.+ */+function decodeIssue(bytes) {+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {+ return bytes.subarray(2).toString("utf16le");+ }+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {+ const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));+ return swapped.swap16().toString("utf16le");+ }+ const text = bytes.toString("utf8");+ return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;+}+function unique(items) {+ return [...new Set(items)];+}+function round(n) {+ return Math.round(n * 1000) / 1000;+}+function cmp(a, b) {+ return a < b ? -1 : a > b ? 1 : 0;+}diff --git a/ix-cli/scripts/ranking-eval/results/README.md b/ix-cli/scripts/ranking-eval/results/README.mdnew file mode 100644index 00000000..2d7e4477--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/README.md@@ -0,0 +1,3 @@+Summary tables of the ranking study. The per-issue records they were computed+from (`*.jsonl`, about 5 MB) are kept out of this repository, in ix-bench's+`results/ranking-eval/`. `eval.mjs` writes new ones here; don't commit them.diff --git a/ix-cli/scripts/ranking-eval/results/dev-ablation.txt b/ix-cli/scripts/ranking-eval/results/dev-ablation.txtnew file mode 100644index 00000000..1303f8cf--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/dev-ablation.txt@@ -0,0 +1,17 @@+metric=bm25 ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+final | 0.600 | 0.737 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.73/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.74/0.75 (12) | 25/9 | 24/5 | 23/3+final_nocode | 0.569 | 0.704 | 0.814 | 0.64/0.75/0.80 (26) | 0.56/0.72/0.88 (44) | 0.54/0.63/0.74 (22) | 0.51/0.67/0.72 (12) | 23/11 | 17/5 | 18/2+final_nostop | 0.557 | 0.697 | 0.784 | 0.66/0.72/0.75 (26) | 0.54/0.70/0.82 (44) | 0.49/0.67/0.80 (22) | 0.53/0.67/0.72 (12) | 19/7 | 15/2 | 14/2+final_noid | 0.603 | 0.727 | 0.817 | 0.69/0.75/0.81 (26) | 0.59/0.78/0.88 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 23/7 | 19/4 | 17/1+final_nosite | 0.579 | 0.725 | 0.829 | 0.70/0.75/0.82 (26) | 0.53/0.71/0.87 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/10 | 22/6 | 21/3++metric=plan ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+final | 0.632 | 0.761 | 0.848 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.89 (44) | 0.80/0.84/0.84 (22) | 0.56/0.74/0.75 (12) | 17/8 | 16/5 | 16/3+final_nocode | 0.639 | 0.753 | 0.836 | 0.68/0.76/0.81 (26) | 0.56/0.72/0.88 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 16/7 | 12/4 | 13/2+final_nostop | 0.603 | 0.733 | 0.793 | 0.64/0.72/0.75 (26) | 0.52/0.70/0.82 (44) | 0.75/0.84/0.84 (22) | 0.53/0.67/0.72 (12) | 10/5 | 9/2 | 8/2+final_noid | 0.620 | 0.744 | 0.836 | 0.64/0.74/0.81 (26) | 0.56/0.75/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 14/7 | 11/5 | 11/1+final_nosite | 0.621 | 0.749 | 0.829 | 0.68/0.75/0.82 (26) | 0.51/0.71/0.85 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/9 | 14/6 | 14/3diff --git a/ix-cli/scripts/ranking-eval/results/dev-r1.txt b/ix-cli/scripts/ranking-eval/results/dev-r1.txtnew file mode 100644index 00000000..fbdb7024--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/dev-r1.txt@@ -0,0 +1,55 @@+metric=bm25 ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+code | 0.559 | 0.646 | 0.762 | 0.61/0.69/0.73 (26) | 0.58/0.70/0.83 (44) | 0.52/0.54/0.70 (22) | 0.45/0.56/0.69 (12) | 13/1 | 6/1 | 8/1+stop | 0.540 | 0.691 | 0.786 | 0.61/0.69/0.80 (26) | 0.54/0.75/0.87 (44) | 0.47/0.61/0.65 (22) | 0.51/0.61/0.70 (12) | 15/8 | 13/4 | 13/2+min4 | 0.494 | 0.607 | 0.732 | 0.42/0.59/0.70 (26) | 0.55/0.68/0.81 (44) | 0.45/0.52/0.63 (22) | 0.53/0.56/0.72 (12) | 7/7 | 2/4 | 3/2+qtf | 0.535 | 0.665 | 0.726 | 0.52/0.65/0.69 (26) | 0.55/0.73/0.81 (44) | 0.52/0.61/0.63 (22) | 0.53/0.56/0.69 (12) | 11/5 | 7/1 | 5/4+qtf8 | 0.535 | 0.664 | 0.764 | 0.54/0.61/0.78 (26) | 0.53/0.74/0.83 (44) | 0.52/0.63/0.66 (22) | 0.56/0.56/0.69 (12) | 19/10 | 12/4 | 12/3+id2 | 0.555 | 0.670 | 0.728 | 0.56/0.67/0.71 (26) | 0.58/0.72/0.80 (44) | 0.52/0.58/0.63 (22) | 0.53/0.65/0.69 (12) | 17/5 | 9/1 | 4/1+id3 | 0.534 | 0.652 | 0.757 | 0.56/0.68/0.76 (26) | 0.52/0.66/0.81 (44) | 0.52/0.58/0.67 (22) | 0.53/0.69/0.72 (12) | 17/9 | 12/4 | 8/1+id5 | 0.527 | 0.671 | 0.765 | 0.54/0.71/0.77 (26) | 0.53/0.68/0.80 (44) | 0.47/0.58/0.72 (22) | 0.59/0.72/0.72 (12) | 20/11 | 15/6 | 13/6+id3p | 0.519 | 0.637 | 0.741 | 0.56/0.68/0.76 (26) | 0.51/0.64/0.79 (44) | 0.47/0.56/0.63 (22) | 0.53/0.69/0.72 (12) | 16/10 | 11/5 | 7/3+path1 | 0.496 | 0.617 | 0.730 | 0.45/0.61/0.71 (26) | 0.58/0.69/0.82 (44) | 0.45/0.52/0.61 (22) | 0.40/0.56/0.69 (12) | 2/1 | 0/1 | 1/0+path2 | 0.505 | 0.612 | 0.728 | 0.47/0.61/0.70 (26) | 0.58/0.68/0.82 (44) | 0.45/0.52/0.61 (22) | 0.40/0.56/0.69 (12) | 3/1 | 0/2 | 0/0+path4 | 0.507 | 0.617 | 0.728 | 0.47/0.61/0.70 (26) | 0.58/0.69/0.82 (44) | 0.45/0.52/0.61 (22) | 0.42/0.56/0.69 (12) | 4/1 | 0/1 | 0/0+path2base | 0.496 | 0.635 | 0.739 | 0.45/0.61/0.70 (26) | 0.58/0.70/0.84 (44) | 0.45/0.52/0.61 (22) | 0.40/0.65/0.69 (12) | 2/1 | 3/2 | 2/0+path4base | 0.496 | 0.635 | 0.739 | 0.45/0.61/0.70 (26) | 0.58/0.70/0.84 (44) | 0.45/0.52/0.61 (22) | 0.40/0.65/0.69 (12) | 2/1 | 3/2 | 2/0+k1_09 | 0.493 | 0.611 | 0.717 | 0.42/0.56/0.70 (26) | 0.59/0.72/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.51/0.69 (12) | 1/4 | 1/5 | 1/3+k1_15 | 0.520 | 0.615 | 0.717 | 0.51/0.61/0.69 (26) | 0.58/0.68/0.82 (44) | 0.47/0.52/0.56 (22) | 0.40/0.56/0.69 (12) | 6/2 | 2/3 | 1/2+k1_20 | 0.526 | 0.626 | 0.726 | 0.53/0.63/0.70 (26) | 0.57/0.69/0.81 (44) | 0.42/0.52/0.61 (22) | 0.53/0.56/0.72 (12) | 10/4 | 5/3 | 5/3+b03 | 0.506 | 0.616 | 0.721 | 0.45/0.54/0.67 (26) | 0.60/0.69/0.82 (44) | 0.47/0.61/0.61 (22) | 0.33/0.51/0.69 (12) | 7/7 | 3/7 | 1/3+b05 | 0.501 | 0.616 | 0.725 | 0.43/0.57/0.68 (26) | 0.61/0.70/0.82 (44) | 0.45/0.56/0.61 (22) | 0.33/0.51/0.69 (12) | 3/5 | 3/6 | 1/1+b09 | 0.503 | 0.606 | 0.729 | 0.48/0.61/0.69 (26) | 0.56/0.66/0.82 (44) | 0.47/0.52/0.61 (22) | 0.42/0.56/0.72 (12) | 6/3 | 2/4 | 4/3+cap3 | 0.506 | 0.611 | 0.723 | 0.41/0.53/0.62 (26) | 0.58/0.69/0.84 (44) | 0.54/0.61/0.65 (22) | 0.38/0.49/0.63 (12) | 8/8 | 3/9 | 3/5+max100 | 0.473 | 0.595 | 0.691 | 0.43/0.62/0.71 (26) | 0.57/0.68/0.79 (44) | 0.38/0.42/0.47 (22) | 0.38/0.56/0.69 (12) | 2/3 | 1/3 | 1/4+nosite | 0.509 | 0.646 | 0.747 | 0.45/0.61/0.70 (26) | 0.60/0.73/0.82 (44) | 0.47/0.56/0.70 (22) | 0.38/0.56/0.69 (12) | 3/0 | 3/0 | 3/0+nodts | 0.497 | 0.619 | 0.726 | 0.43/0.61/0.70 (26) | 0.59/0.69/0.81 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 0/1 | 0/1++metric=plan ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+code | 0.621 | 0.709 | 0.792 | 0.66/0.68/0.74 (26) | 0.58/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 8/1 | 2/1 | 4/2+stop | 0.592 | 0.743 | 0.829 | 0.64/0.75/0.81 (26) | 0.52/0.72/0.87 (44) | 0.71/0.84/0.84 (22) | 0.53/0.63/0.72 (12) | 9/8 | 10/4 | 9/1+min4 | 0.584 | 0.694 | 0.780 | 0.59/0.70/0.75 (26) | 0.56/0.68/0.81 (44) | 0.66/0.80/0.80 (22) | 0.53/0.56/0.72 (12) | 5/5 | 2/2 | 2/2+qtf | 0.594 | 0.719 | 0.773 | 0.64/0.70/0.74 (26) | 0.53/0.73/0.81 (44) | 0.71/0.80/0.80 (22) | 0.53/0.56/0.69 (12) | 7/4 | 4/1 | 3/4+qtf8 | 0.625 | 0.735 | 0.799 | 0.64/0.72/0.77 (26) | 0.55/0.74/0.83 (44) | 0.80/0.84/0.84 (22) | 0.56/0.56/0.69 (12) | 13/6 | 9/3 | 9/4+id2 | 0.629 | 0.732 | 0.781 | 0.65/0.70/0.74 (26) | 0.56/0.72/0.80 (44) | 0.80/0.84/0.84 (22) | 0.53/0.65/0.69 (12) | 12/4 | 5/1 | 3/2+id3 | 0.623 | 0.714 | 0.787 | 0.64/0.71/0.75 (26) | 0.55/0.66/0.80 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 11/6 | 8/4 | 5/2+id5 | 0.626 | 0.725 | 0.795 | 0.67/0.71/0.76 (26) | 0.53/0.68/0.79 (44) | 0.78/0.84/0.89 (22) | 0.59/0.72/0.72 (12) | 16/8 | 11/5 | 10/7+id3p | 0.618 | 0.705 | 0.781 | 0.64/0.71/0.75 (26) | 0.54/0.64/0.79 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 11/7 | 8/5 | 5/4+path1 | 0.584 | 0.690 | 0.778 | 0.60/0.68/0.74 (26) | 0.58/0.68/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 0/2 | 0/1+path2 | 0.581 | 0.690 | 0.775 | 0.59/0.68/0.74 (26) | 0.58/0.68/0.81 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/2 | 0/2+path4 | 0.581 | 0.695 | 0.778 | 0.59/0.68/0.74 (26) | 0.58/0.69/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/1 | 0/1+path2base | 0.585 | 0.711 | 0.789 | 0.61/0.69/0.74 (26) | 0.58/0.70/0.84 (44) | 0.66/0.80/0.80 (22) | 0.42/0.65/0.69 (12) | 2/1 | 3/2 | 2/1+path4base | 0.585 | 0.711 | 0.789 | 0.61/0.69/0.74 (26) | 0.58/0.70/0.84 (44) | 0.66/0.80/0.80 (22) | 0.42/0.65/0.69 (12) | 2/1 | 3/2 | 2/1+k1_09 | 0.586 | 0.695 | 0.770 | 0.58/0.68/0.75 (26) | 0.60/0.70/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.53/0.69 (12) | 1/2 | 0/2 | 1/3+k1_15 | 0.607 | 0.702 | 0.787 | 0.64/0.70/0.74 (26) | 0.58/0.67/0.82 (44) | 0.71/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 4/0 | 3/3 | 1/1+k1_20 | 0.611 | 0.707 | 0.787 | 0.64/0.70/0.74 (26) | 0.57/0.68/0.81 (44) | 0.71/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 6/2 | 4/3 | 3/2+b03 | 0.610 | 0.695 | 0.775 | 0.61/0.68/0.74 (26) | 0.60/0.70/0.82 (44) | 0.75/0.80/0.80 (22) | 0.38/0.53/0.66 (12) | 7/6 | 1/4 | 1/3+b05 | 0.593 | 0.697 | 0.783 | 0.61/0.69/0.75 (26) | 0.61/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.38/0.53/0.69 (12) | 4/4 | 2/4 | 1/0+b09 | 0.594 | 0.692 | 0.787 | 0.64/0.70/0.73 (26) | 0.56/0.65/0.81 (44) | 0.71/0.84/0.84 (22) | 0.42/0.56/0.72 (12) | 4/2 | 3/4 | 2/3+cap3 | 0.620 | 0.696 | 0.775 | 0.61/0.69/0.73 (26) | 0.62/0.69/0.82 (44) | 0.75/0.80/0.80 (22) | 0.42/0.53/0.66 (12) | 8/5 | 2/4 | 2/3+max100 | 0.562 | 0.674 | 0.752 | 0.59/0.69/0.75 (26) | 0.56/0.68/0.79 (44) | 0.62/0.71/0.71 (22) | 0.42/0.56/0.69 (12) | 0/2 | 1/3 | 0/3+nosite | 0.586 | 0.715 | 0.781 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/0+nodts | 0.581 | 0.698 | 0.779 | 0.59/0.68/0.75 (26) | 0.58/0.69/0.81 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/1 | 0/1diff --git a/ix-cli/scripts/ranking-eval/results/dev-r2.txt b/ix-cli/scripts/ranking-eval/results/dev-r2.txtnew file mode 100644index 00000000..f4774107--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/dev-r2.txt@@ -0,0 +1,37 @@+metric=bm25 ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+code | 0.559 | 0.646 | 0.762 | 0.61/0.69/0.73 (26) | 0.58/0.70/0.83 (44) | 0.52/0.54/0.70 (22) | 0.45/0.56/0.69 (12) | 13/1 | 6/1 | 8/1+stop | 0.540 | 0.691 | 0.786 | 0.61/0.69/0.80 (26) | 0.54/0.75/0.87 (44) | 0.47/0.61/0.65 (22) | 0.51/0.61/0.70 (12) | 15/8 | 13/4 | 13/2+cs | 0.587 | 0.708 | 0.815 | 0.69/0.75/0.81 (26) | 0.57/0.73/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 20/8 | 17/5 | 16/1+cs_q8 | 0.580 | 0.726 | 0.781 | 0.67/0.75/0.80 (26) | 0.54/0.73/0.81 (44) | 0.55/0.71/0.71 (22) | 0.58/0.68/0.77 (12) | 28/16 | 26/7 | 19/7+cs_id2 | 0.579 | 0.725 | 0.829 | 0.70/0.75/0.82 (26) | 0.53/0.71/0.87 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/10 | 22/6 | 21/3+cs_id3 | 0.598 | 0.720 | 0.828 | 0.70/0.75/0.81 (26) | 0.57/0.69/0.88 (44) | 0.54/0.73/0.80 (22) | 0.61/0.74/0.75 (12) | 26/11 | 23/6 | 21/4+cs_q8_id2 | 0.580 | 0.724 | 0.823 | 0.69/0.78/0.83 (26) | 0.53/0.71/0.81 (44) | 0.55/0.68/0.85 (22) | 0.60/0.71/0.80 (12) | 27/14 | 26/8 | 24/7+cs_site | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1+cs_k15 | 0.573 | 0.702 | 0.811 | 0.69/0.76/0.81 (26) | 0.54/0.71/0.87 (44) | 0.54/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 20/9 | 16/5 | 18/2+cs_k20 | 0.544 | 0.707 | 0.824 | 0.70/0.77/0.82 (26) | 0.48/0.71/0.87 (44) | 0.49/0.67/0.80 (22) | 0.51/0.63/0.72 (12) | 21/12 | 17/5 | 20/2+cs_big05 | 0.569 | 0.712 | 0.812 | 0.65/0.76/0.81 (26) | 0.58/0.74/0.88 (44) | 0.49/0.67/0.74 (22) | 0.51/0.58/0.69 (12) | 19/8 | 17/4 | 15/1+cs_big1 | 0.558 | 0.708 | 0.813 | 0.65/0.76/0.81 (26) | 0.56/0.74/0.88 (44) | 0.49/0.67/0.74 (22) | 0.47/0.54/0.69 (12) | 19/9 | 16/4 | 15/1+cs_path2base | 0.589 | 0.723 | 0.809 | 0.69/0.75/0.81 (26) | 0.57/0.75/0.87 (44) | 0.54/0.67/0.74 (22) | 0.51/0.67/0.70 (12) | 21/8 | 18/4 | 16/2+max256 | 0.498 | 0.632 | 0.739 | 0.43/0.61/0.70 (26) | 0.56/0.69/0.81 (44) | 0.51/0.58/0.67 (22) | 0.38/0.56/0.69 (12) | 2/1 | 2/1 | 2/1+code_big1 | 0.560 | 0.658 | 0.763 | 0.63/0.70/0.73 (26) | 0.57/0.70/0.83 (44) | 0.52/0.58/0.70 (22) | 0.45/0.56/0.69 (12) | 16/3 | 7/0 | 9/1++metric=plan ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+code | 0.621 | 0.709 | 0.792 | 0.66/0.68/0.74 (26) | 0.58/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 8/1 | 2/1 | 4/2+stop | 0.592 | 0.743 | 0.829 | 0.64/0.75/0.81 (26) | 0.52/0.72/0.87 (44) | 0.71/0.84/0.84 (22) | 0.53/0.63/0.72 (12) | 9/8 | 10/4 | 9/1+cs | 0.613 | 0.725 | 0.836 | 0.64/0.74/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 12/8 | 9/6 | 11/1+cs_q8 | 0.635 | 0.756 | 0.809 | 0.70/0.76/0.80 (26) | 0.53/0.73/0.81 (44) | 0.80/0.84/0.84 (22) | 0.58/0.68/0.76 (12) | 19/12 | 19/7 | 14/7+cs_id2 | 0.621 | 0.749 | 0.829 | 0.68/0.75/0.82 (26) | 0.51/0.71/0.85 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/9 | 14/6 | 14/3+cs_id3 | 0.627 | 0.742 | 0.828 | 0.70/0.75/0.81 (26) | 0.53/0.69/0.85 (44) | 0.78/0.84/0.84 (22) | 0.53/0.74/0.75 (12) | 19/11 | 16/7 | 15/5+cs_q8_id2 | 0.637 | 0.748 | 0.832 | 0.73/0.78/0.83 (26) | 0.51/0.71/0.81 (44) | 0.80/0.80/0.90 (22) | 0.60/0.71/0.80 (12) | 21/11 | 19/8 | 19/7+cs_site | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1+cs_k15 | 0.590 | 0.723 | 0.829 | 0.65/0.75/0.81 (26) | 0.49/0.69/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 11/9 | 7/4 | 11/2+cs_k20 | 0.585 | 0.730 | 0.833 | 0.70/0.75/0.82 (26) | 0.45/0.70/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.59/0.72 (12) | 13/10 | 9/6 | 12/2+cs_big05 | 0.617 | 0.729 | 0.833 | 0.65/0.75/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 14/8 | 9/5 | 10/1+cs_big1 | 0.605 | 0.729 | 0.834 | 0.65/0.75/0.81 (26) | 0.53/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.49/0.56/0.69 (12) | 14/9 | 9/5 | 10/1+cs_path2base | 0.613 | 0.737 | 0.829 | 0.64/0.74/0.81 (26) | 0.55/0.71/0.87 (44) | 0.75/0.84/0.84 (22) | 0.53/0.65/0.69 (12) | 12/8 | 10/6 | 10/2+max256 | 0.584 | 0.703 | 0.782 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.81 (44) | 0.68/0.81/0.81 (22) | 0.42/0.56/0.69 (12) | 1/0 | 1/0 | 1/1+code_big1 | 0.600 | 0.711 | 0.793 | 0.62/0.69/0.74 (26) | 0.55/0.70/0.83 (44) | 0.75/0.84/0.84 (22) | 0.45/0.56/0.69 (12) | 9/3 | 2/0 | 5/2diff --git a/ix-cli/scripts/ranking-eval/results/dev-r3.txt b/ix-cli/scripts/ranking-eval/results/dev-r3.txtnew file mode 100644index 00000000..641d4c31--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/dev-r3.txt@@ -0,0 +1,33 @@+metric=bm25 ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+cs | 0.587 | 0.708 | 0.815 | 0.69/0.75/0.81 (26) | 0.57/0.73/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 20/8 | 17/5 | 16/1+cs_site | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1+cs_site0 | 0.613 | 0.727 | 0.826 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 24/7 | 19/4 | 18/1+cs_siteonly | 0.603 | 0.727 | 0.824 | 0.69/0.75/0.81 (26) | 0.61/0.78/0.91 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 23/7 | 19/4 | 17/1+css_id2 | 0.633 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.64/0.76/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 29/9 | 24/5 | 23/3+css_id3 | 0.629 | 0.751 | 0.837 | 0.70/0.75/0.81 (26) | 0.61/0.77/0.90 (44) | 0.59/0.73/0.80 (22) | 0.61/0.74/0.75 (12) | 30/10 | 26/5 | 22/4+css_q8 | 0.604 | 0.736 | 0.791 | 0.67/0.75/0.80 (26) | 0.58/0.75/0.83 (44) | 0.59/0.71/0.71 (22) | 0.58/0.68/0.77 (12) | 31/16 | 27/7 | 19/6+css_q2 | 0.624 | 0.732 | 0.826 | 0.71/0.75/0.82 (26) | 0.60/0.76/0.88 (44) | 0.64/0.68/0.75 (22) | 0.52/0.66/0.77 (12) | 28/10 | 24/4 | 23/3+css_k15 | 0.609 | 0.724 | 0.821 | 0.69/0.76/0.81 (26) | 0.60/0.77/0.89 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.72 (12) | 25/8 | 18/3 | 19/2+css_k10 | 0.614 | 0.739 | 0.823 | 0.68/0.74/0.81 (26) | 0.61/0.81/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.69 (12) | 23/6 | 20/3 | 17/1+css_b06 | 0.615 | 0.741 | 0.820 | 0.68/0.73/0.80 (26) | 0.61/0.80/0.91 (44) | 0.63/0.72/0.74 (22) | 0.47/0.58/0.69 (12) | 23/7 | 20/4 | 16/1+css_b09 | 0.609 | 0.730 | 0.817 | 0.69/0.76/0.79 (26) | 0.60/0.77/0.88 (44) | 0.58/0.68/0.75 (22) | 0.51/0.63/0.75 (12) | 25/8 | 19/4 | 19/3+css_big05 | 0.605 | 0.730 | 0.823 | 0.65/0.76/0.81 (26) | 0.61/0.78/0.91 (44) | 0.58/0.67/0.75 (22) | 0.51/0.58/0.69 (12) | 24/7 | 19/3 | 17/1++metric=plan ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+cs | 0.613 | 0.725 | 0.836 | 0.64/0.74/0.81 (26) | 0.55/0.70/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 12/8 | 9/6 | 11/1+cs_site | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1+cs_site0 | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1+cs_siteonly | 0.630 | 0.744 | 0.846 | 0.64/0.74/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/7 | 11/5 | 12/1+css_id2 | 0.656 | 0.768 | 0.850 | 0.68/0.75/0.82 (26) | 0.60/0.76/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 20/8 | 16/5 | 17/3+css_id3 | 0.656 | 0.775 | 0.837 | 0.70/0.75/0.81 (26) | 0.60/0.77/0.88 (44) | 0.78/0.84/0.84 (22) | 0.53/0.74/0.75 (12) | 22/10 | 19/5 | 16/5+css_q8 | 0.659 | 0.766 | 0.818 | 0.70/0.76/0.80 (26) | 0.59/0.75/0.83 (44) | 0.80/0.84/0.84 (22) | 0.58/0.68/0.76 (12) | 22/12 | 20/7 | 14/6+css_q2 | 0.632 | 0.764 | 0.846 | 0.70/0.75/0.83 (26) | 0.56/0.76/0.88 (44) | 0.75/0.84/0.84 (22) | 0.52/0.66/0.77 (12) | 18/10 | 16/4 | 18/3+css_k15 | 0.616 | 0.755 | 0.839 | 0.65/0.75/0.81 (26) | 0.55/0.77/0.89 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 15/8 | 10/2 | 12/2+css_k10 | 0.630 | 0.771 | 0.842 | 0.64/0.73/0.81 (26) | 0.59/0.81/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 15/7 | 12/2 | 11/1+css_b06 | 0.622 | 0.767 | 0.842 | 0.64/0.74/0.81 (26) | 0.58/0.80/0.91 (44) | 0.75/0.84/0.84 (22) | 0.49/0.56/0.69 (12) | 15/9 | 13/3 | 11/1+css_b09 | 0.616 | 0.763 | 0.836 | 0.65/0.74/0.79 (26) | 0.55/0.77/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.65/0.75 (12) | 15/8 | 11/3 | 13/3+css_big05 | 0.633 | 0.747 | 0.842 | 0.65/0.75/0.81 (26) | 0.59/0.75/0.91 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.69 (12) | 17/7 | 11/4 | 11/1diff --git a/ix-cli/scripts/ranking-eval/results/dev-v2.txt b/ix-cli/scripts/ranking-eval/results/dev-v2.txtnew file mode 100644index 00000000..3ba54c20--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/dev-v2.txt@@ -0,0 +1,32 @@+metric=bm25 ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.494 | 0.622 | 0.728 | 0.43/0.61/0.70 (26) | 0.59/0.70/0.82 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+picks | 0.499 | 0.636 | 0.719 | 0.43/0.61/0.70 (26) | 0.60/0.73/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 2/0 | 0/1+picks_base | 0.499 | 0.636 | 0.719 | 0.43/0.61/0.70 (26) | 0.60/0.73/0.79 (44) | 0.45/0.52/0.61 (22) | 0.38/0.56/0.69 (12) | 1/0 | 2/0 | 0/1+final | 0.600 | 0.737 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.73/0.90 (44) | 0.58/0.73/0.80 (22) | 0.56/0.74/0.75 (12) | 25/9 | 24/5 | 23/3+final2 | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 24/9 | 24/5 | 23/3+final2_nocode | 0.560 | 0.713 | 0.804 | 0.62/0.75/0.80 (26) | 0.56/0.75/0.88 (44) | 0.52/0.63/0.70 (22) | 0.51/0.67/0.72 (12) | 22/11 | 18/5 | 17/2+final2_nostop | 0.557 | 0.697 | 0.784 | 0.66/0.72/0.75 (26) | 0.54/0.73/0.82 (44) | 0.49/0.63/0.80 (22) | 0.53/0.67/0.72 (12) | 19/7 | 15/2 | 14/2+final2_noid | 0.593 | 0.727 | 0.815 | 0.69/0.75/0.81 (26) | 0.59/0.78/0.88 (44) | 0.54/0.67/0.74 (22) | 0.51/0.58/0.72 (12) | 22/7 | 19/4 | 16/1+impl | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 24/9 | 24/5 | 23/3++metric=plan ref=base n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.581 | 0.700 | 0.781 | 0.59/0.68/0.75 (26) | 0.58/0.70/0.82 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 0/0 | 0/0 | 0/0+picks | 0.586 | 0.715 | 0.771 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/1+picks_base | 0.586 | 0.715 | 0.771 | 0.59/0.68/0.75 (26) | 0.59/0.73/0.79 (44) | 0.66/0.80/0.80 (22) | 0.42/0.56/0.69 (12) | 1/0 | 2/0 | 0/1+final | 0.632 | 0.761 | 0.848 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.89 (44) | 0.80/0.84/0.84 (22) | 0.56/0.74/0.75 (12) | 17/8 | 16/5 | 16/3+final2 | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 17/8 | 15/5 | 17/3+final2_nocode | 0.639 | 0.753 | 0.836 | 0.68/0.76/0.81 (26) | 0.56/0.72/0.88 (44) | 0.80/0.84/0.84 (22) | 0.53/0.69/0.72 (12) | 16/7 | 12/4 | 13/2+final2_nostop | 0.603 | 0.733 | 0.793 | 0.64/0.72/0.75 (26) | 0.52/0.70/0.82 (44) | 0.75/0.84/0.84 (22) | 0.53/0.67/0.72 (12) | 10/5 | 9/2 | 8/2+final2_noid | 0.620 | 0.744 | 0.836 | 0.64/0.74/0.81 (26) | 0.56/0.75/0.88 (44) | 0.75/0.84/0.84 (22) | 0.53/0.56/0.72 (12) | 14/7 | 11/5 | 11/1+impl | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 17/8 | 15/5 | 17/3++metric=bm25 ref=picks n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+final2 | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/9 | 22/5 | 24/3+impl | 0.590 | 0.744 | 0.841 | 0.70/0.75/0.82 (26) | 0.56/0.76/0.90 (44) | 0.54/0.73/0.80 (22) | 0.56/0.72/0.75 (12) | 23/9 | 22/5 | 24/3+metric=plan ref=picks n=104+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+final2 | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/8 | 14/6 | 18/3+impl | 0.632 | 0.759 | 0.850 | 0.68/0.75/0.82 (26) | 0.54/0.73/0.90 (44) | 0.80/0.84/0.84 (22) | 0.56/0.72/0.75 (12) | 16/8 | 14/6 | 18/3diff --git a/ix-cli/scripts/ranking-eval/results/test-final.txt b/ix-cli/scripts/ranking-eval/results/test-final.txtnew file mode 100644index 00000000..09a3dde0--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/test-final.txt@@ -0,0 +1,9 @@+metric=bm25 ref=base n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.470 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.49/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 0/0 | 0/0 | 0/0+impl | 0.601 | 0.660 | 0.733 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.76/0.82/0.84 (15) | 0.53/0.57/0.66 (15) | 13/3 | 10/3 | 6/4++metric=plan ref=base n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.517 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.42/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 0/0 | 0/0 | 0/0+impl | 0.601 | 0.676 | 0.750 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.77/0.82/0.84 (15) | 0.52/0.57/0.66 (15) | 10/4 | 9/2 | 5/4diff --git a/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txt b/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txtnew file mode 100644index 00000000..01b136f9--- /dev/null+++ b/ix-cli/scripts/ranking-eval/results/test-v2-second-scoring.txt@@ -0,0 +1,22 @@+SECOND scoring of the test split, after a post-hoc fix: the first final variant's+guides|demos?|benchmarks? exclusions dropped files real fixes change, so isSourcePath+was replaced by fix/issue-start-picking's list. First scoring: test-final.txt.++metric=bm25 ref=base n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.470 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.49/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 0/0 | 0/0 | 0/0+picks | 0.495 | 0.598 | 0.692 | 0.47/0.54/0.61 (15) | 0.59/0.66/0.78 (15) | 0.54/0.68/0.77 (15) | 0.38/0.52/0.61 (15) | 2/0 | 0/0 | 0/0+impl | 0.606 | 0.671 | 0.744 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.78/0.87/0.89 (15) | 0.53/0.57/0.66 (15) | 14/2 | 11/2 | 7/3++metric=plan ref=base n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+base | 0.517 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.42/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 0/0 | 0/0 | 0/0+picks | 0.542 | 0.626 | 0.728 | 0.52/0.54/0.68 (15) | 0.52/0.66/0.78 (15) | 0.72/0.79/0.84 (15) | 0.40/0.52/0.61 (15) | 2/0 | 0/0 | 0/0+impl | 0.606 | 0.687 | 0.761 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.79/0.87/0.89 (15) | 0.52/0.57/0.66 (15) | 10/3 | 10/1 | 6/3++metric=bm25 ref=picks n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+impl | 0.606 | 0.671 | 0.744 | 0.48/0.52/0.59 (15) | 0.63/0.73/0.84 (15) | 0.78/0.87/0.89 (15) | 0.53/0.57/0.66 (15) | 13/3 | 11/2 | 7/3+metric=plan ref=picks n=60+variant | @5 | @10 | @20 | Java @5/@10/@20 | JS @5/@10/@20 | Py @5/@10/@20 | TS @5/@10/@20 | W/L @5 | W/L @10 | W/L @20+impl | 0.606 | 0.687 | 0.761 | 0.55/0.58/0.65 (15) | 0.57/0.73/0.84 (15) | 0.79/0.87/0.89 (15) | 0.52/0.57/0.66 (15) | 9/4 | 10/1 | 6/3diff --git a/ix-cli/scripts/ranking-eval/speed.mjs b/ix-cli/scripts/ranking-eval/speed.mjsnew file mode 100644index 00000000..fb524856--- /dev/null+++ b/ix-cli/scripts/ranking-eval/speed.mjs@@ -0,0 +1,48 @@+// Time per issue of the baseline and the current bm25Rank, over the same+// files and issue text: median of 7 runs after one warm-up.+//+// node speed.mjs <instance_id>[,<instance_id>...] [--worktree <dir>]+//+// Without --worktree, files are read from memory (the snapshot, preloaded),+// so the time is tokenising and scoring alone. With --worktree <checkout>,+// files are read from disk through gitRepoAccess, as `ix context+// --from-issue` reads them. With --at <owner/repo>@<commit>, the dev issue's+// text is ranked against that snapshot instead (timing only, no scoring).+import { readFileSync } from "node:fs";++import * as dist from "../../dist/cli/explain/issue.js";+import { gitRepoAccess } from "../../dist/cli/explain/text-references.js";+import * as base from "./baseline-issue.mjs";+import { loadCorpus } from "./corpus.mjs";++const argv = process.argv.slice(2);+const ids = new Set(argv[0].split(","));+const wtIdx = argv.indexOf("--worktree");+const worktree = wtIdx >= 0 ? argv[wtIdx + 1] : undefined;+const atIdx = argv.indexOf("--at");+const at = atIdx >= 0 ? argv[atIdx + 1].split("@") : undefined;+const rows = readFileSync("/home/ihock/src/ix-bench/results/external/polybench_verified.jsonl", "utf8")+ .split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((r) => ids.has(r.instance_id));++function time(fn) {+ fn();+ const ms = [];+ for (let i = 0; i < 7; i++) {+ const t = process.hrtime.bigint();+ fn();+ ms.push(Number(process.hrtime.bigint() - t) / 1e6);+ }+ return ms.sort((a, b) => a - b)[3];+}++for (const row of rows) {+ const repo = worktree ? gitRepoAccess(worktree) : at ? loadCorpus(at[0], at[1]) : loadCorpus(row.repo, row.base_commit);+ const files = repo.files();+ const q = row.problem_statement;+ const bFiles = files.filter(base.isSourcePath);+ const nFiles = files.filter(dist.isSourcePath);+ const tb = time(() => base.bm25Rank(repo, bFiles, q));+ const tn = time(() => dist.bm25Rank(repo, nFiles, q));+ console.log(`${row.instance_id}${at ? ` on ${at[0]}` : ""} ${worktree ? "disk" : "memory"} files base=${bFiles.length} new=${nFiles.length} `+ + `issue=${q.length}ch base ${tb.toFixed(0)} ms new ${tn.toFixed(0)} ms`);+}diff --git a/ix-cli/scripts/ranking-eval/split.json b/ix-cli/scripts/ranking-eval/split.jsonnew file mode 100644index 00000000..416e98ef--- /dev/null+++ b/ix-cli/scripts/ranking-eval/split.json@@ -0,0 +1,171 @@+{+ "seed": 20260930,+ "dev": [+ "apache__dubbo-3093",+ "apache__dubbo-3317",+ "apache__dubbo-3622",+ "apache__dubbo-3855",+ "apache__dubbo-4026",+ "apache__rocketmq-1636",+ "apache__rocketmq-4122",+ "apache__rocketmq-5008",+ "apache__rocketmq-5037",+ "apache__rocketmq-5062",+ "apolloconfig__apollo-1894",+ "apolloconfig__apollo-4207",+ "apolloconfig__apollo-4464",+ "apolloconfig__apollo-4568",+ "coder__code-server-3277",+ "coder__code-server-4923",+ "coder__code-server-6278",+ "google__gson-1703",+ "google__gson-1904",+ "google__gson-1989",+ "google__gson-2060",+ "google__gson-2071",+ "google__gson-2134",+ "google__gson-2158",+ "google__gson-2167",+ "google__gson-2337",+ "google__gson-2475",+ "google__gson-2479",+ "google__gson-2498",+ "keras-team__keras-18553",+ "keras-team__keras-18871",+ "keras-team__keras-18975",+ "keras-team__keras-19190",+ "keras-team__keras-19201",+ "keras-team__keras-19459",+ "keras-team__keras-19466",+ "keras-team__keras-19484",+ "keras-team__keras-19775",+ "keras-team__keras-19826",+ "keras-team__keras-19838",+ "keras-team__keras-19844",+ "langchain-ai__langchain-19331",+ "langchain-ai__langchain-20064",+ "langchain-ai__langchain-3367",+ "langchain-ai__langchain-4009",+ "langchain-ai__langchain-4103",+ "mrdoob__three.js-14836",+ "mrdoob__three.js-18648",+ "mrdoob__three.js-20991",+ "mrdoob__three.js-24461",+ "mui__material-ui-11451",+ "mui__material-ui-11858",+ "mui__material-ui-11987",+ "mui__material-ui-12236",+ "mui__material-ui-13534",+ "mui__material-ui-13690",+ "prettier__prettier-11000",+ "prettier__prettier-11637",+ "prettier__prettier-12930",+ "prettier__prettier-14400",+ "prettier__prettier-3436",+ "prettier__prettier-3515",+ "prettier__prettier-361",+ "prettier__prettier-3723",+ "prettier__prettier-459",+ "prettier__prettier-666",+ "prettier__prettier-8046",+ "prettier__prettier-8777",+ "serverless__serverless-1910",+ "serverless__serverless-2014",+ "serverless__serverless-2434",+ "serverless__serverless-2576",+ "serverless__serverless-2842",+ "serverless__serverless-2945",+ "serverless__serverless-2952",+ "serverless__serverless-4192",+ "serverless__serverless-5571",+ "serverless__serverless-5640",+ "serverless__serverless-5775",+ "serverless__serverless-5842",+ "sveltejs__svelte-1049",+ "sveltejs__svelte-1095",+ "sveltejs__svelte-1116",+ "sveltejs__svelte-1137",+ "sveltejs__svelte-1190",+ "sveltejs__svelte-1227",+ "sveltejs__svelte-1231",+ "sveltejs__svelte-1310",+ "sveltejs__svelte-1376",+ "sveltejs__svelte-1923",+ "sveltejs__svelte-1932",+ "sveltejs__svelte-3702",+ "sveltejs__svelte-3749",+ "sveltejs__svelte-3949",+ "sveltejs__svelte-4146",+ "sveltejs__svelte-4288",+ "tailwindlabs__tailwindcss-116",+ "tailwindlabs__tailwindcss-550",+ "tailwindlabs__tailwindcss-853",+ "yt-dlp__yt-dlp-10390",+ "yt-dlp__yt-dlp-4841",+ "yt-dlp__yt-dlp-5195",+ "yt-dlp__yt-dlp-5933",+ "yt-dlp__yt-dlp-9862"+ ],+ "test": [+ "apache__dubbo-4515",+ "apache__dubbo-4567",+ "apache__dubbo-5356",+ "apache__rocketmq-7420",+ "apache__rocketmq-7563",+ "apache__rocketmq-7712",+ "google__gson-2410",+ "google__gson-2420",+ "google__gson-2435",+ "google__gson-2701",+ "huggingface__transformers-16661",+ "huggingface__transformers-22158",+ "huggingface__transformers-23126",+ "huggingface__transformers-26568",+ "huggingface__transformers-26752",+ "huggingface__transformers-28517",+ "huggingface__transformers-30556",+ "huggingface__transformers-30934",+ "huggingface__transformers-6098",+ "huggingface__transformers-7075",+ "huggingface__transformers-7374",+ "huggingface__transformers-7562",+ "huggingface__transformers-8435",+ "huggingface__transformers-8624",+ "keras-team__keras-19937",+ "microsoft__vscode-106767",+ "microsoft__vscode-110094",+ "microsoft__vscode-113837",+ "microsoft__vscode-160342",+ "microsoft__vscode-189223",+ "mui__material-ui-14638",+ "mui__material-ui-15359",+ "mui__material-ui-18141",+ "mui__material-ui-19511",+ "mui__material-ui-24105",+ "mui__material-ui-25874",+ "mui__material-ui-26061",+ "mui__material-ui-26807",+ "mui__material-ui-34610",+ "mui__material-ui-36971",+ "prettier__prettier-4667",+ "prettier__prettier-6604",+ "prettier__prettier-9850",+ "serverless__serverless-3095",+ "serverless__serverless-5860",+ "serverless__serverless-6534",+ "serverless__serverless-7102",+ "serverless__serverless-7374",+ "serverless__serverless-7617",+ "serverless__serverless-8159",+ "sveltejs__svelte-3403",+ "sveltejs__svelte-4332",+ "sveltejs__svelte-510",+ "sveltejs__svelte-5875",+ "sveltejs__svelte-6759",+ "trinodb__trino-1512",+ "trinodb__trino-2707",+ "trinodb__trino-2768",+ "trinodb__trino-3603",+ "trinodb__trino-4393"+ ]+}\ No newline at end of filediff --git a/ix-cli/scripts/ranking-eval/summarize.mjs b/ix-cli/scripts/ranking-eval/summarize.mjsnew file mode 100644index 00000000..b123a41e--- /dev/null+++ b/ix-cli/scripts/ranking-eval/summarize.mjs@@ -0,0 +1,45 @@+// Tables from eval.mjs output: mean recall @5/@10/@20 per language and+// overall, and per-instance wins/losses against a reference variant (a win is+// a higher recall at that cutoff).+//+// node summarize.mjs results/dev-*.jsonl [--ref base] [--metric bm25|plan] [--md]+import { readFileSync } from "node:fs";++const argv = process.argv.slice(2);+const opt = (name, dflt) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };+const ref = opt("ref", "base");+const metric = opt("metric", "bm25");+const files = argv.filter((a, i) => !a.startsWith("--") && !argv[i - 1]?.startsWith("--"));+const recs = files.flatMap((f) => readFileSync(f, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)))+ .filter((r) => !r.skipped && r[metric]);+const byVariant = new Map();+for (const r of recs) {+ if (!byVariant.has(r.variant)) byVariant.set(r.variant, new Map());+ byVariant.get(r.variant).set(r.id, r);+}+const langs = [...new Set(recs.map((r) => r.lang))].sort();+const mean = (xs) => xs.reduce((a, b) => a + b, 0) / (xs.length || 1);+const f3 = (x) => x.toFixed(3);+const refMap = byVariant.get(ref);+console.log(`metric=${metric} ref=${ref} n=${refMap?.size}`);+const head = ["variant", "@5", "@10", "@20", ...langs.map((l) => `${({ JavaScript: "JS", TypeScript: "TS", Python: "Py", Java: "Java" })[l] ?? l} @5/@10/@20`), "W/L @5", "W/L @10", "W/L @20"];+console.log(head.join(" | "));+for (const [v, m] of byVariant) {+ const rs = [...m.values()];+ const cells = [v, ...[0, 1, 2].map((k) => f3(mean(rs.map((r) => r[metric][k]))))];+ for (const l of langs) {+ const lr = rs.filter((r) => r.lang === l);+ cells.push(`${[0, 1, 2].map((k) => mean(lr.map((r) => r[metric][k])).toFixed(2)).join("/")} (${lr.length})`);+ }+ for (const k of [0, 1, 2]) {+ let w = 0, l = 0;+ for (const r of rs) {+ const b = refMap?.get(r.id);+ if (!b) continue;+ if (r[metric][k] > b[metric][k] + 1e-9) w++;+ else if (r[metric][k] < b[metric][k] - 1e-9) l++;+ }+ cells.push(`${w}/${l}`);+ }+ console.log(cells.join(" | "));+}diff --git a/ix-cli/scripts/ranking-eval/variants.mjs b/ix-cli/scripts/ranking-eval/variants.mjsnew file mode 100644index 00000000..7bae8352--- /dev/null+++ b/ix-cli/scripts/ranking-eval/variants.mjs@@ -0,0 +1,162 @@+// BM25 variants for the ranking study, one knob at a time. `base` reproduces+// issue.ts's bm25Rank exactly (checked by eval.mjs --check); the rest change+// one thing each, so a gain can be attributed.+import { extractCandidates } from "./baseline-issue.mjs";++const WORD_BASE = /[A-Za-z][A-Za-z0-9]*/g;+const WORD_CODE = /[A-Za-z_][A-Za-z0-9_]*/g;+const PART = /[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g;++// Issue-prose words that say nothing about where the code is. English+// function words, plus the words bug reports are written in.+export const STOPWORDS = new Set((+ "the and for are but not you all any can had her was one our out has him his how its may new now old see "+ + "two way who did get let say she too use about above after again also been before being below between both "+ + "could does doing down during each few from further have having here into just more most much must only other "+ + "over own same should some such than that their them then there these they this those through under until very "+ + "want what when where which while whom why will with would your yours "+ + "bug bugs issue issues expected behavior behaviour actual reproduce reproduction steps version versions "+ + "current currently describe description problem thanks thank please like think seems seem happen happens "+ + "work works working instead because since using used example following sure able still even though"+).split(/\s+/));++/** Tokenizer: `base` is issue.ts's; `code` also keeps a multi-part identifier whole. */+export function tokenize(text, o = {}) {+ const minLen = o.minLen ?? 3;+ const out = [];+ if (!o.code) {+ for (const word of text.match(WORD_BASE) ?? []) {+ const parts = word.match(PART) ?? [word];+ for (const part of parts) if (part.length >= minLen) out.push(part.toLowerCase());+ }+ return out;+ }+ for (const word of text.match(WORD_CODE) ?? []) {+ const parts = word.match(PART) ?? [];+ let n = 0;+ for (const part of parts) {+ n++;+ if (part.length >= minLen) out.push(part.toLowerCase());+ }+ if (o.whole !== false && n >= 2) {+ const whole = word.replace(/_/g, "").toLowerCase();+ if (whole.length >= minLen) out.push(whole);+ }+ }+ return out;+}++function queryWeights(query, o) {+ const toks = tokenize(query, o).filter((t) => !(o.stop && STOPWORDS.has(t)));+ const qtf = new Map();+ for (const t of toks) qtf.set(t, (qtf.get(t) ?? 0) + 1);+ const w = new Map();+ for (const [t, f] of qtf) {+ // BM25's query-term saturation (k3): a word the issue repeats counts more.+ w.set(t, o.k3 ? ((o.k3 + 1) * f) / (o.k3 + f) : 1);+ }+ if (o.bigram) {+ // "save model" in prose asks for `saveModel` / `save_model`, which the+ // code tokenizer keeps whole as `savemodel`.+ const seq = tokenize(query, { minLen: o.minLen });+ for (let i = 0; i + 1 < seq.length; i++) {+ if (STOPWORDS.has(seq[i]) || STOPWORDS.has(seq[i + 1])) continue;+ const t = seq[i] + seq[i + 1];+ if (!w.has(t)) w.set(t, o.bigram);+ }+ }+ if (o.idW && o.idW !== 1) {+ const { paths, identifiers } = extractCandidates(query);+ const idTerms = new Set();+ for (const id of [...identifiers, ...(o.idPaths ? paths : [])]) for (const t of tokenize(id, o)) idTerms.add(t);+ for (const t of idTerms) if (w.has(t)) w.set(t, w.get(t) * o.idW);+ }+ return w;+}++/**+ * Options:+ * code, whole, minLen tokenizer+ * stop drop STOPWORDS from the query+ * k1, b BM25+ * k3 query term frequency saturation (0: off, as base)+ * idW, idPaths weight of query terms from identifiers extractCandidates finds+ * pathW, pathB, pathField BM25F path field: weight, length norm, "full" | "base" (0: path is body text, as base)+ * maxBytes skip files larger than this (base: 200KB)+ * lenCap cap a doc's length at lenCap * avg in normalisation+ * penalty (path) => multiplier on the final score+ * exclude RegExp: paths left out of the corpus, as isSourcePath would+ */+export function rank(repo, files, query, o = {}) {+ const k1 = o.k1 ?? 1.2;+ const b = o.b ?? 0.75;+ const maxBytes = o.maxBytes ?? 200 * 1024;+ const qw = queryWeights(query, o);+ const terms = [...qw.keys()];+ const wanted = new Set(terms);+ const docs = [];+ for (const path of files) {+ if (o.exclude?.test(path)) continue;+ const text = repo.read(path);+ if (text === undefined || Buffer.byteLength(text, "utf8") > maxBytes) continue;+ const tf = new Map();+ let length;+ let ptf;+ let plen = 0;+ if (!o.pathW) {+ const tokens = tokenize(`${path.replace(/\//g, " ")} ${text}`, o);+ for (const t of tokens) if (wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);+ length = tokens.length;+ } else {+ const tokens = tokenize(text, o);+ for (const t of tokens) if (wanted.has(t)) tf.set(t, (tf.get(t) ?? 0) + 1);+ length = tokens.length;+ const ptext = o.pathField === "base" ? path.slice(path.lastIndexOf("/") + 1) : path.replace(/\//g, " ");+ const ptoks = tokenize(ptext.replace(/\.[A-Za-z]+$/, ""), o);+ ptf = new Map();+ for (const t of ptoks) if (wanted.has(t)) ptf.set(t, (ptf.get(t) ?? 0) + 1);+ plen = ptoks.length;+ }+ docs.push({ path, tf, ptf, length, plen });+ }+ if (docs.length === 0) return [];+ const n = docs.length;+ let total = 0;+ let ptotal = 0;+ const df = new Map();+ for (const d of docs) {+ total += d.length;+ ptotal += d.plen;+ const seen = new Set(d.tf.keys());+ if (d.ptf) for (const t of d.ptf.keys()) seen.add(t);+ for (const t of seen) df.set(t, (df.get(t) ?? 0) + 1);+ }+ const avg = total / n || 1;+ const pavg = ptotal / n || 1;+ const pathB = o.pathB ?? 0.5;+ const out = [];+ for (const d of docs) {+ const len = o.lenCap ? Math.min(d.length, o.lenCap * avg) : d.length;+ const norm = 1 - b + (b * len) / avg;+ const pnorm = 1 - pathB + (pathB * d.plen) / pavg;+ let score = 0;+ for (const t of terms) {+ const f = d.tf.get(t) ?? 0;+ const pf = d.ptf?.get(t) ?? 0;+ if (!f && !pf) continue;+ const dd = df.get(t);+ const idf = Math.log(1 + (n - dd + 0.5) / (dd + 0.5));+ let s;+ if (!o.pathW) {+ s = (f * (k1 + 1)) / (f + k1 * norm);+ } else {+ const tt = f / norm + (o.pathW * pf) / pnorm; // BM25F: fields combined before saturation+ s = (tt * (k1 + 1)) / (tt + k1);+ }+ score += qw.get(t) * idf * s;+ }+ if (o.penalty) score *= o.penalty(d.path);+ if (score > 0) out.push({ path: d.path, score });+ }+ return out.sort((a, c) => c.score - a.score || (a.path < c.path ? -1 : a.path > c.path ? 1 : 0));+}