Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

20.5 KB
Raw Blame
// Copyright 2026 Ix Infrastructure Inc.
// issue.ts of fix/issue-start-picking (b62b524), the branch this one is stacked on, frozen as a
// second baseline: its wider isSourcePath and name order, the old BM25. Transpiled; imports repointed.
import { readFileSync } from "node:fs";
import { posix } from "node:path";
import { isTestPath } from "../../dist/cli/explain/related-files.js";
import { resolveToken } from "../../dist/cli/explain/text-references.js";
/**
 * From an issue's text to the files its fix starts in, for
 * `ix context --from-issue`.
 *
 * Measured on SWE-PolyBench (28 fresh instances, not used while building
 * this): resolving the code names an issue mentions to their definitions put
 * a changed file in the top five 0.600 of the time, against 0.485 for BM25
 * over the repository. But a graph-expanded bundle, ranked by graph distance,
 * lost to BM25 at equal size (0.57 against 0.67). So the ranking here is
 * lexical first: the resolved starting files lead, BM25 against the issue
 * orders the rest, and graph closeness to a starting point only breaks ties
 * and nudges.
 *
 * Starting points, most specific first:
 *
 *  1. file paths the issue names, resolved against the tracked files;
 *  2. multi-part identifiers -- `newJsonWriter`, `save_model`, `GsonBuilder`
 *     -- from backticks or running text. A multi-part name is almost always a
 *     code name;
 *  3. single backticked words of four letters or more. `save` was the right
 *     start for a keras fix; `flex`, `code` and `version` were not, which is
 *     why a single word counts only in backticks, and only after the rest.
 *
 * A name resolves only to an exact-name definition in source code. The first
 * pilot run started from a CSS output file, a changelog, test fixtures, and an
 * import: `borderStylesReset` resolved to the file that imported it, because
 * the graph's `module` entity for an import is located in the importing file.
 */
/** Starting points a bundle is built from. */
export const MAX_STARTS = 3;
/** Candidate names looked up, in order. */
export const MAX_CANDIDATES = 25;
/** Files larger than this are not scored: generated, bundled or data. */
export const MAX_BM25_BYTES = 200 * 1024;
/** Ranked files carried in the JSON bundle. */
export const MAX_RANKED_FILES = 20;
/**
 * How much being next to a starting point is worth, as a fraction of a file's
 * BM25 score. Small on purpose: on the benchmark, graph order lost to BM25 at
 * equal size, so closeness may reorder files the text scores alike and may
 * not overturn a clear lexical lead.
 */
export const CLOSENESS_BOOST = 0.1;
/** Extensions of files a fix edits. */
const CODE_EXTENSIONS = [
    "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "py", "java", "svelte", "vue", "go", "rs",
    "kt", "kts", "scala", "rb", "php", "cs", "c", "h", "cc", "cpp", "hpp", "swift",
];
const CODE_FILE = new RegExp(`\\.(?:${CODE_EXTENSIONS.join("|")})$`);
/**
 * Not code a fix edits: tests, test data, samples, docs, vendored or built
 * output. The pilot's list, plus what `isTestPath` already knows.
 *
 * The second line is a project's website, playground, test configuration,
 * benchmarks, stories and end-to-end suites. Svelte's `site/` alone was 40
 * of the wrong files in Ix's top-10 rankings over two samples of issues; none
 * of these directories holds any of the 932 files the 382 SWE-PolyBench
 * Verified fixes change.
 */
const NOISE = new RegExp("(^|/)(__tests__|tests?|spec|specs|fixtures?|__fixtures__|test-fixtures|samples|examples?|vendor|"
    + "third_party|node_modules|dist|build|coverage|docs?|changelog_unreleased|"
    + "site|website|docs-site|playground|tests?[_-]config|benchmarks|\\.?storybook|e2e|cypress|integration-tests)/"
    + "|\\.(test|spec)\\.[^/]+$|\\.min\\.js$|(^|/)test_[^/]+\\.py$|_test\\.(py|go)$");
/** Source code a fix would edit: a code file that is not a test, fixture, doc or build output. */
export function isSourcePath(path) {
    return CODE_FILE.test(path) && !NOISE.test(path) && !isTestPath(path);
}
// A path-like token with a code extension: `saving_api.py`, `src/a/index.ts`,
// and the absolute path of a stack trace frame or the tail of a repository URL,
// `/home/u/proj/src/a.py` and `//github.com/o/r/blob/main/src/a.py`, which
// `resolveIssuePath` cuts down to the tracked path they end in.
//
// The lookbehind refuses `.` and `-` as well as `\w` and `/`: a match can then
// only begin where a run of path characters begins, so a long unbroken run --
// a pasted UUID list, a row of dots -- is scanned once, not once per `.`.
const PATH = new RegExp(`(?<![\\w/.-])(\\/*(?:[\\w.-]+/)*[\\w.-]+\\.(?:${CODE_EXTENSIONS.join("|")}))\\b`, "g");
const BACKTICK = /`([^`\n]{2,80})`/g;
const WORD = /[A-Za-z_][A-Za-z0-9_]{2,}/g;
// camelCase, PascalCase with two or more parts, snake_case, SCREAMING_SNAKE.
const IDENT_SOURCE = "(?:[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[a-z]+(?:[A-Z][a-z0-9]*)+|[a-z0-9]+(?:_[a-z0-9]+)+|[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+)";
const IDENT = new RegExp(`\\b${IDENT_SOURCE}\\b`, "g");
const IDENT_FULL = new RegExp(`^${IDENT_SOURCE}$`);
/** The paths and code names an issue mentions, most specific first. */
export function extractCandidates(text) {
    const paths = unique([...text.matchAll(PATH)].map((m) => m[1]));
    // A path's own parts are not names in the issue: `saving_api` in
    // `saving/saving_api.py` is the path again, not a function.
    const prose = text.replace(PATH, " ");
    const inTicks = [...prose.matchAll(BACKTICK)].flatMap((m) => [...m[1].matchAll(WORD)].map((w) => w[0]));
    const multi = [...inTicks, ...[...prose.matchAll(IDENT)].map((m) => m[0])].filter((t) => IDENT_FULL.test(t));
    const single = inTicks.filter((t) => !IDENT_FULL.test(t) && t.length >= 4);
    // Only MAX_STARTS names become starts, so order decides: a short common
    // word (`repeat`, `ignore`) mentioned first took a slot a specific one
    // (`Stylesheet`) mentioned later needed. Common words still resolve, last.
    const ordered = [
        ...multi,
        ...single.filter((t) => !isGenericIdentifier(t)),
        ...single.filter((t) => isGenericIdentifier(t)),
    ];
    const seen = new Set(paths);
    const identifiers = [];
    for (const token of ordered) {
        if (seen.has(token))
            continue;
        seen.add(token);
        identifiers.push(token);
    }
    return { paths, identifiers: identifiers.slice(0, MAX_CANDIDATES) };
}
/** Kinds a definition has, most navigable first. Anything else ranks after. */
const DEFINITION_KINDS = [
    "class", "interface", "trait", "struct", "enum", "type", "object", "function", "method",
    "constant", "variable", "property", "field",
];
/** Never a definition: an import (located in the importer), a chunk, a file. */
const NOT_DEFINITIONS = new Set(["module", "chunk", "file"]);
function kindRank(kind) {
    const i = DEFINITION_KINDS.indexOf(kind.toLowerCase());
    return i === -1 ? DEFINITION_KINDS.length : i;
}
/**
 * Up to `max` starting points, first found wins, one per file; and the
 * candidates that resolved to nothing. Candidates after the last start are
 * not looked up, and not reported as unresolved.
 */
export async function pickStartingPoints(text, deps, max = MAX_STARTS) {
    const { paths, identifiers } = extractCandidates(text);
    const starts = [];
    const unresolved = [];
    const taken = (path) => starts.some((s) => s.path === path);
    const tracked = new Set(deps.files);
    const byBasename = new Map();
    for (const file of deps.files) {
        const base = posix.basename(file);
        byBasename.set(base, [...(byBasename.get(base) ?? []), file]);
    }
    for (const token of paths) {
        if (starts.length >= max)
            return { starts, unresolved };
        const path = resolveIssuePath(token, tracked, byBasename);
        if (!path || !isSourcePath(path)) {
            unresolved.push(token);
            continue;
        }
        if (!taken(path))
            starts.push({ token, name: posix.basename(path), kind: "file", path, via: "path in issue" });
    }
    const score = deps.fileScore ?? (() => 0);
    for (const token of identifiers) {
        if (starts.length >= max)
            break;
        // Not caught: a backend that cannot answer is an error to report, not an
        // issue that names nothing.
        const hits = (await deps.search(token))
            .filter((h) => h.name === token && !!h.path && !NOT_DEFINITIONS.has(h.kind.toLowerCase()) && isSourcePath(h.path))
            .sort((a, b) => kindRank(a.kind) - kindRank(b.kind) || score(b.path) - score(a.path) || cmp(a.path, b.path));
        if (hits.length === 0) {
            unresolved.push(token);
            continue;
        }
        const best = hits[0];
        if (taken(best.path))
            continue;
        starts.push({
            token, id: best.id, name: best.name, kind: best.kind, path: best.path,
            ...(best.lineStart !== undefined ? { lineStart: best.lineStart } : {}),
            ...(best.lineEnd !== undefined ? { lineEnd: best.lineEnd } : {}),
            via: "identifier in issue",
        });
    }
    return { starts, unresolved };
}
/**
 * A path as an issue writes it, to the tracked file it names. First as
 * `resolveToken` reads a path in code; then, for an absolute path from a
 * stack trace or the tail of a repository URL, the longest of its suffixes
 * that is a tracked path: `/home/u/proj/src/a.py` and
 * `//github.com/o/r/blob/main/src/a.py` both name `src/a.py`.
 */
function resolveIssuePath(token, tracked, byBasename) {
    const direct = resolveToken(token, "", tracked, byBasename);
    if (direct)
        return direct;
    const parts = token.split("/");
    for (let i = 1; i < parts.length; i++) {
        const suffix = parts.slice(i).join("/");
        if (tracked.has(suffix))
            return suffix;
    }
    return undefined;
}
/**
 * Words for BM25: identifiers split at camelCase and underscores, lowercased,
 * parts of two letters or fewer dropped. `listByKind` is `list kind`, so an
 * issue that says "list by kind" matches the code that spells it as one word.
 */
export function bm25Tokens(text) {
    const out = [];
    for (const word of text.match(/[A-Za-z][A-Za-z0-9]*/g) ?? []) {
        const parts = word.match(/[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])/g) ?? [word];
        for (const part of parts)
            if (part.length > 2)
                out.push(part.toLowerCase());
    }
    return out;
}
/**
 * Okapi BM25 (k1 1.2, b 0.75) of each file against `query`, best first, files
 * that share no word with it left out. A file's path is part of its text, so
 * `auth/login.ts` matches an issue about logging in before it is opened.
 */
export function bm25Rank(repo, files, query, k1 = 1.2, b = 0.75) {
    // Only the query's words are counted: BM25 needs no other term frequency,
    // and a map of every word of every file is most of the memory and a third
    // of the time on a large repository. A document's length is still every word.
    const terms = [...new Set(bm25Tokens(query))];
    const wanted = new Set(terms);
    const docs = new Map();
    for (const path of files) {
        const text = repo.read(path);
        if (text === undefined || Buffer.byteLength(text, "utf8") > MAX_BM25_BYTES)
            continue;
        const tf = new Map();
        const tokens = bm25Tokens(`${path.replace(/\//g, " ")} ${text}`);
        for (const t of tokens)
            if (wanted.has(t))
                tf.set(t, (tf.get(t) ?? 0) + 1);
        docs.set(path, { tf, length: tokens.length });
    }
    if (docs.size === 0)
        return [];
    const n = docs.size;
    let total = 0;
    const df = new Map();
    for (const doc of docs.values()) {
        total += doc.length;
        for (const t of doc.tf.keys())
            df.set(t, (df.get(t) ?? 0) + 1);
    }
    const avg = total / n || 1;
    const out = [];
    for (const [path, doc] of docs) {
        let score = 0;
        for (const t of terms) {
            const f = doc.tf.get(t);
            if (!f)
                continue;
            const d = df.get(t);
            const idf = Math.log(1 + (n - d + 0.5) / (d + 0.5));
            score += idf * (f * (k1 + 1)) / (f + k1 * (1 - b + b * doc.length / avg));
        }
        if (score > 0)
            out.push({ path, score });
    }
    return out.sort((a, c) => c.score - a.score || cmp(a.path, c.path));
}
/**
 * Starting points and the lexical ranking for one issue. Without a git
 * checkout there is no file list: paths do not resolve, BM25 is empty, and
 * only names can start the bundle.
 */
export async function planIssue(text, deps) {
    const files = deps.repo?.files() ?? [];
    const bm25 = deps.repo ? bm25Rank(deps.repo, files.filter(isSourcePath), text) : [];
    const scores = new Map(bm25.map((h) => [h.path, h.score]));
    const { starts, unresolved } = await pickStartingPoints(text, {
        files, search: deps.search, fileScore: (p) => scores.get(p) ?? 0,
    });
    if (starts.length > 0 || bm25.length === 0)
        return { starts, unresolved, fallback: false, bm25 };
    const top = bm25[0].path;
    return {
        starts: [{ token: top, name: posix.basename(top), kind: "file", path: top, via: "bm25 fallback" }],
        unresolved,
        fallback: true,
        bm25,
    };
}
/**
 * Whether an identifier the issue names is too common to point anywhere: a
 * short all-lowercase word such as `debug`, `bind`, `config` or `render`
 * resolves to *a* definition, rarely the one the fix touches. Specific names
 * carry an inner capital (`toJsonTree`, `JsonWriter`), an underscore
 * (`get_openai_callback`), or length.
 *
 * Fixed before it was scored on held-out samples: on SWE-PolyBench issues the
 * starts it keeps were in the gold patch 20 times in 32, the ones it drops 2
 * in 7.
 */
export function isGenericIdentifier(token) {
    return !(/[a-z][A-Z]/.test(token) || /^[A-Z][a-z]+[A-Z]/.test(token) || token.includes("_") || token.length >= 8);
}
/**
 * A plan is confident when the issue named a path, or a specific identifier,
 * that resolved. A BM25 fallback never is: across four samples of
 * SWE-PolyBench issues its start was in the gold patch once in 55.
 */
export function issueConfidence(plan) {
    if (plan.fallback) {
        return { confident: false, starts: [], reason: "no code name in the issue resolved to a definition" };
    }
    const starts = plan.starts.filter((s) => s.via === "path in issue"
        || (s.via === "identifier in issue" && !isGenericIdentifier(s.token)));
    if (starts.length === 0) {
        const names = plan.starts.map((s) => s.token);
        return {
            confident: false,
            starts,
            reason: names.length > 0
                ? `only common words resolved (${names.join(", ")}), which name many definitions`
                : "nothing in the issue resolved to a definition",
        };
    }
    return { confident: true, starts };
}
/** Files after the starting points in a lean view. */
export const LEAN_RANKED = 4;
/**
 * The lean view of a plan. A full bundle costs about 1.5k tokens, and an
 * agent re-reads it on every turn; on SWE-PolyBench it added a median 14%
 * tokens even where its start was right, and saved no turns. What the agent
 * uses is the pointer, so that is all this keeps: the trusted starting points
 * and a few ranked files -- Ix's starts, then BM25, the ordering that beat
 * both alone on held-out issues. When nothing is trusted it says so, rather
 * than pointing the agent at a guess.
 */
export function leanIssueView(plan, ranked = LEAN_RANKED) {
    const c = issueConfidence(plan);
    if (!c.confident) {
        return { confidence: "low", reason: c.reason, startingPoints: [], alsoRanked: [], unresolved: plan.unresolved };
    }
    const startPaths = new Set(c.starts.map((s) => s.path));
    const alsoRanked = rankIssueFiles({ starts: [], bm25: plan.bm25, near: new Map() }, ranked + startPaths.size)
        .filter((f) => !startPaths.has(f.path))
        .slice(0, ranked);
    return { confidence: "high", startingPoints: c.starts, alsoRanked, unresolved: plan.unresolved };
}
/** BM25 files tried, best first, for one the graph has a node for. */
export const CENTRE_FALLBACK_TRIES = 10;
/**
 * The starting point a bundle is centred on: the first start the graph has a
 * node for, else the best BM25 file that has one.
 *
 * A start can have no node: the graph does not index every tracked file.
 * On SWE-PolyBench two svelte issues named no code, BM25's top file was a
 * `.svelte` component, and the command failed with no bundle at all while
 * the fifth-ranked file was a perfectly good `.js` module. Walking BM25's
 * ranking finds that one; the files the walk skipped still rank as usual.
 * `walked` says the centre came from this walk, which is a fallback too.
 */
export async function chooseCentre(starts, bm25, findNode, tries = CENTRE_FALLBACK_TRIES) {
    const located = await Promise.all(starts.map(async (s) => {
        const id = s.id ?? (await findNode(s.path));
        return id ? { ...s, id } : s;
    }));
    const found = located.find((s) => s.id);
    if (found)
        return { starts: located, centre: found, walked: false };
    const tried = new Set(located.map((s) => s.path));
    for (const hit of bm25.filter((h) => !tried.has(h.path)).slice(0, tries)) {
        const id = await findNode(hit.path);
        if (!id)
            continue;
        const centre = {
            token: hit.path, id, name: posix.basename(hit.path), kind: "file", path: hit.path, via: "bm25 fallback",
        };
        return { starts: [...located, centre], centre, walked: true };
    }
    return { starts: located, walked: false };
}
/**
 * The files to read, best first: the starting files in the order they were
 * found, then every other source file by
 *
 *     bm25(file, issue) * (1 + CLOSENESS_BOOST * closeness(file))
 *
 * where closeness is 1 for a file one hop from a starting point, 0.5 for one
 * of its ranked related files, 0 otherwise. Ties go to the closer file, then
 * the path. A graph neighbour the issue's text does not match at all still
 * ranks, after every file it does match.
 */
export function rankIssueFiles(input, limit = MAX_RANKED_FILES) {
    const lexical = new Map(input.bm25.map((h) => [h.path, h.score]));
    const out = [];
    const startPaths = new Set();
    for (const s of input.starts) {
        if (startPaths.has(s.path))
            continue;
        startPaths.add(s.path);
        out.push({
            path: s.path,
            score: round(lexical.get(s.path) ?? 0),
            reason: `starting point (${s.via}: ${s.token})`,
        });
    }
    const pool = new Set([...lexical.keys(), ...input.near.keys()]);
    const rest = [...pool]
        .filter((path) => !startPaths.has(path) && isSourcePath(path))
        .map((path) => {
        const bm25 = lexical.get(path) ?? 0;
        const near = input.near.get(path);
        const closeness = near?.weight ?? 0;
        const reason = [bm25 > 0 ? `bm25 ${bm25.toFixed(2)}` : "no bm25 match", near?.reason]
            .filter(Boolean).join("; ");
        return { path, score: bm25 * (1 + CLOSENESS_BOOST * closeness), closeness, reason };
    })
        .sort((a, b) => b.score - a.score || b.closeness - a.closeness || cmp(a.path, b.path));
    for (const r of rest)
        out.push({ path: r.path, score: round(r.score), reason: r.reason });
    return out.slice(0, limit);
}
/**
 * The issue's text: a file, or stdin for `-`. `stdin` is a parameter so the
 * stream can be replaced under test.
 */
export async function readIssueText(arg, stdin = process.stdin) {
    if (arg !== "-")
        return decodeIssue(readFileSync(arg));
    const chunks = [];
    for await (const chunk of stdin) {
        chunks.push(typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk));
    }
    return decodeIssue(Buffer.concat(chunks));
}
/**
 * UTF-8, unless a byte-order mark says UTF-16. Windows PowerShell 5.1's `>`
 * writes UTF-16LE with a BOM, so `gh issue view 1 --json body -q .body >
 * issue.md` there produces a file that, read as UTF-8, is every letter
 * followed by a NUL: no name, no path, and no BM25 word in it.
 */
function decodeIssue(bytes) {
    if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
        return bytes.subarray(2).toString("utf16le");
    }
    if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
        const swapped = Buffer.from(bytes.subarray(2, bytes.length - (bytes.length % 2)));
        return swapped.swap16().toString("utf16le");
    }
    const text = bytes.toString("utf8");
    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
}
function unique(items) {
    return [...new Set(items)];
}
function round(n) {
    return Math.round(n * 1000) / 1000;
}
function cmp(a, b) {
    return a < b ? -1 : a > b ? 1 : 0;
}