Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

fix(ask): rank multi-scope repos on comparable scores (#117) (#126)

Vivien Perrellea477a80f211e
diff --git a/src/ask/ask.ts b/src/ask/ask.ts
index c4d8d474..2be6aa8c 100644
--- a/src/ask/ask.ts
+++ b/src/ask/ask.ts
@@ -532,12 +532,22 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
// branch is a byte-level regression guarantee for single-scope repos.
const scopes = graph ? scopesOfGraph(graph) : null;
let scopeMeta: AskResult["scopes"];
+ // The BM25 length prior, corpus-global like `idf`/`dfltIdf` above. Hoisted out
+ // of the single-scope branch so BOTH paths score against the same statistics
+ // — that shared basis is half of what makes multi-scope scores comparable
+ // (see fuse.ts's module header). Under `--in`, the sidecar's `avgBodyLen`
+ // covers the whole corpus and must not be reused for the filtered set.
+ const avgBodyLen = askIndex && !inPrefix
+ ? askIndex.avgBodyLen
+ : symbolDocs.length
+ ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length
+ : 0;
if (scopes && scopes.length > 1) {
- // ── Multi-scope repo: rank each sub-project against its OWN corpus (its
- // IDF, its BM25 length prior, its subgraph walk), then fuse the per-scope
- // orderings by reciprocal rank — the big scope can't drown the small one.
- // Same scoring functions and blend as the single-scope path, just fed
- // per-scope inputs; the fusion itself lives in fuse.ts. ──
+ // ── Multi-scope repo: partition by sub-project and walk each subgraph
+ // separately — so the big scope can't drown the small one — but score every
+ // scope against the repo-wide statistics, so the results stay comparable and
+ // can be ordered by score. Same scoring functions and blend as the
+ // single-scope path; the combination itself lives in fuse.ts. ──
const byScope = new Map<string, typeof symbolDocs>();
for (const d of symbolDocs) {
const s = scopeOf(d.n.path, scopes).prefix;
@@ -545,47 +555,35 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
if (list) list.push(d);
else byScope.set(s, [d]);
}
- const idfOf = new Map<string, { idf: Map<string, number>; dflt: number }>();
const fusion = rankScopesAndFuse(
[...byScope.keys()].sort(),
{
lex: (s) => {
const docs = byScope.get(s)!;
- // Per-scope IDF/BM25 stats: concepts are repo-level so their bags
- // fold in everywhere, symbol bags come from this scope only. (The
- // build sidecar's df/avgBodyLen are corpus-global, so multi-scope
- // always computes these live — from the sidecar's own token maps.)
- const sIdf = computeIdf([
- ...conceptBags,
- ...docs.map((d) => new Set([...d.name.keys(), ...d.path.keys(), ...d.body.keys()])),
- ]);
- idfOf.set(s, { idf: sIdf, dflt: Math.log(1 + conceptBags.length + docs.length) });
- const avg = docs.reduce((a, d) => a + bodyLen(d.body), 0) / docs.length;
+ // Repo-wide IDF and length prior — the SAME `idf`/`avgBodyLen` the
+ // single-scope path uses. Scoring each scope against its own corpus
+ // (as this did before #117) made a term worth a different amount in
+ // every scope, so the resulting scores could not be compared and the
+ // combined order had to fall back on rank. Only the PARTITION is
+ // per-scope now; the statistics are global.
const out = new Map<string, number>();
for (const { n, name, path, body } of docs) {
const total =
- (score(q, name, sIdf) * 3 +
- score(q, path, sIdf) * 2 +
- bm25(q, body, sIdf, bodyLen(body), avg)) *
+ (score(q, name, idf) * 3 +
+ score(q, path, idf) * 2 +
+ bm25(q, body, idf, bodyLen(body), avgBodyLen)) *
testFactor(n.path);
if (total > 0) out.set(n.id, total);
}
return out;
},
- // The participation gate's match-STRENGTH signal, evaluated once per
- // scope on that scope's raw-lex top doc — same `strongShare`/
- // `matchedIdfShare` functions and per-scope idf `lex` (above) just
- // computed, so this reads the SAME strength `federateAsk` gates on
- // (see fuse.ts's STRONG_FLOOR/HIGH_FLOOR).
- strength: (s, id) => {
- const d = docsById.get(id);
- const si = idfOf.get(s);
- if (!d || !si) return { coverage: 0, coverageStrong: 0 };
- return {
- coverage: matchedIdfShare(q, [d.name, d.path, d.body], si.idf, si.dflt),
- coverageStrong: strongShare(q, d.n, d, si.idf, si.dflt),
- };
- },
+ // Body-only suppression: does any doc in this scope carry a query term
+ // in an identifier (name) or its path, rather than only in prose?
+ hasIdentifierMatch: (s) =>
+ (byScope.get(s) ?? []).some(({ name, path }) => {
+ for (const t of q.keys()) if (hasTerm(name, t) || hasTerm(path, t)) return true;
+ return false;
+ }),
walk: (s, seeds) =>
graphRank
? personalizedPageRank(graph!, seeds, {
@@ -612,9 +610,11 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
scope: rd.scope,
};
const d = docsById.get(rd.id);
- const si = idfOf.get(rd.scope);
- matchedOf.set(hit, d && si ? matchedIdfShare(q, [d.name, d.path, d.body], si.idf, si.dflt) : 0);
- matchedStrongOf.set(hit, d && si ? strongShare(q, n, d, si.idf, si.dflt) : 0);
+ // Global idf here too, so per-hit coverage means the same thing in a
+ // multi-scope repo as it does in a single-scope one — the escalation
+ // nudge and the hook's coverage gate both read these.
+ matchedOf.set(hit, d ? matchedIdfShare(q, [d.name, d.path, d.body], idf, dfltIdf) : 0);
+ matchedStrongOf.set(hit, d ? strongShare(q, n, d, idf, dfltIdf) : 0);
symbolHits.push(hit);
}
// Label + footer only when federation actually happened (or a scope was
@@ -625,14 +625,8 @@ function lexical(query: string, corpus: Corpus, limit: number, graphRank: boolea
} else {
// Single-scope: the pre-scopes ranking path, byte-identical output when
// `--in` is absent (Task 3's regression guarantee: `askIndex && !inPrefix`
- // reduces to plain `askIndex` when `inPrefix` is undefined, so this line is
- // a no-op change for every non-`--in` caller). Under `--in`, `askIndex`'s
- // `avgBodyLen` is corpus-global and must not be reused for the filtered set.
- const avgBodyLen = askIndex && !inPrefix
- ? askIndex.avgBodyLen
- : symbolDocs.length
- ? symbolDocs.reduce((a, d) => a + bodyLen(d.body), 0) / symbolDocs.length
- : 0;
+ // reduces to plain `askIndex` when `inPrefix` is undefined). `avgBodyLen` is
+ // now defined once above and shared with the multi-scope branch.
const lex = new Map<string, number>(); // node id → lexical score (>0 only)
let maxLex = 0;
for (const { n, name, path, body } of symbolDocs) {
diff --git a/src/ask/fuse.ts b/src/ask/fuse.ts
index 3e627634..fc0ca299 100644
--- a/src/ask/fuse.ts
+++ b/src/ask/fuse.ts
@@ -1,27 +1,44 @@
/**
- * Scope-aware ranking for `graft ask` — per-scope ranking + reciprocal-rank
- * fusion (RRF). Pure functions, no fs.
+ * Scope-aware ranking for `graft ask`. Pure functions, no fs.
*
* The problem: lexical + graph scores are corpus-relative. In a multi-scope
* repo (a monorepo's `frontend/` + `backend/`, sub-projects under one root),
- * pooling every node into one ranking lets the biggest sub-project set the
- * score scale and drown the small one — its IDF, its BM25 length prior, its
- * PageRank mass. The fix: rank each scope against ITS OWN corpus (per-scope
- * IDF/BM25/walk, done by the caller), then fuse the per-scope ORDERINGS here
- * with RRF — rank positions are comparable across corpora where raw scores
- * are not, so the tiny scope's best hit lands next to the huge scope's best.
+ * scoring every node against one pooled corpus lets the biggest sub-project set
+ * the score scale and drown the small one.
+ *
+ * This module solves it by keeping the scope PARTITION — each scope is scored
+ * and walked separately — while making the resulting scores COMPARABLE, so the
+ * combined order can be read straight off the score. Comparability has two
+ * requirements, and both must hold or the order is meaningless:
+ *
+ * 1. shared corpus statistics — the caller scores every scope against the
+ * repo-wide IDF and length prior (see `ask.ts`), so the same term is worth
+ * the same everywhere;
+ * 2. a shared normalization denominator — {@link rankScopesAndFuse} divides
+ * every scope by the best raw score in the REPO, not by each scope's own
+ * best (see `globalMaxLex` there).
+ *
+ * Issue #117 was the consequence of neither holding. Each scope divided by its
+ * own maximum, so a barely-relevant scope's top hit normalized to ~1.0 exactly
+ * like the genuinely relevant scope's top hit; RRF then fused by RANK, which
+ * discards magnitude outright, so the weak scope's rank-1 tied the strong
+ * scope's rank-1 and displaced its rank-2 — frequently the answer. Measurement
+ * across four repositories and four ecosystems put 73% of missed files in that
+ * bucket. RRF is retained for `federateAsk` (see `graph/workspace.ts`), where
+ * the corpora really are separate repositories and no shared denominator
+ * exists — that is the case rank fusion is for.
*
* Soft federation: a scope whose top hit is probably an incidental word
- * collision, not a second home for the query, is left out of the fused order
+ * collision, not a second home for the query, is left out of the combined order
* and reported in `alsoMatched` instead, so the output can say "also matched:
- * docs/ — narrow with --in docs/" rather than diluting the pack. The REAL
- * participation gate — used by both `rankScopesAndFuse` here and `federateAsk`
- * (workspace federation) — is the match-STRENGTH signal in {@link
- * STRONG_FLOOR}/{@link HIGH_FLOOR}: a scope's top hit must have matched a
- * query term in a NAME/PATH field, or have broad enough overall coverage to be
- * real even body-only. `fuseScopes`'s own {@link PARTICIPATION_RATIO} gate is
- * a separate, cruder ratio check that still runs internally as a
- * post-normalization safety net, not the primary defense.
+ * docs/ — narrow with --in docs/" rather than diluting the pack. With
+ * comparable scores that gate is simply {@link PARTICIPATION_RATIO} — a scope
+ * must reach a real share of the best score anywhere in the repo. Under
+ * per-scope normalization that check was vacuous (every scope's top was ~1.0),
+ * which is why a separate match-STRENGTH gate ({@link STRONG_FLOOR}/{@link
+ * HIGH_FLOOR}) had to stand in for it; that signal is still exported and still
+ * used by `federateAsk`, which normalizes per child repo and therefore still
+ * needs it.
*/
/** One scored document, attributed to the ranking scope (path prefix, "" =
@@ -160,22 +177,100 @@ export function fuseScopes(docs: ScopedDoc[]): FusionResult {
return { ranked, federated, alsoMatched };
}
+/**
+ * Combine scopes whose scores are ALREADY comparable (shared corpus statistics
+ * and a shared normalization denominator — see the module header). Same shape
+ * and same gate as {@link fuseScopes}, but the combined order is read straight
+ * off the score instead of from reciprocal rank, because with a common scale
+ * the magnitude is exactly the signal rank fusion would throw away.
+ *
+ * Only {@link rankScopesAndFuse} may call this. `federateAsk` fuses separate
+ * repositories, which have no common denominator, and must keep using
+ * {@link fuseScopes}.
+ */
+export function combineComparableScopes(docs: ScopedDoc[]): FusionResult {
+ const byScope = new Map<string, ScopedDoc[]>();
+ for (const d of docs) {
+ if (d.score <= 0) continue;
+ const list = byScope.get(d.scope);
+ if (list) list.push(d);
+ else byScope.set(d.scope, [d]);
+ }
+ for (const list of byScope.values()) list.sort(byScore);
+
+ if (byScope.size === 0) return { ranked: [], federated: [], alsoMatched: [] };
+ // One scoring scope: pass through untouched. Identical to the single-scope
+ // path by construction — the guarantee that a single-scope repo cannot move.
+ if (byScope.size === 1) {
+ const [scope, list] = byScope.entries().next().value as [string, ScopedDoc[]];
+ return {
+ ranked: list.map((d) => ({ id: d.id, scope, score: d.score })),
+ federated: [scope],
+ alsoMatched: [],
+ };
+ }
+
+ // Gate on a real share of the best score in the repo. This is the same
+ // ratio check `fuseScopes` runs, but here it MEANS something: the scores
+ // being compared share a denominator.
+ const scopesByBest = [...byScope.entries()].sort(
+ (a, b) => b[1][0].score - a[1][0].score || a[0].localeCompare(b[0]),
+ );
+ const gate = PARTICIPATION_RATIO * scopesByBest[0][1][0].score;
+ const federated: string[] = [];
+ const alsoMatched: FusionResult["alsoMatched"] = [];
+ for (const [scope, list] of scopesByBest) {
+ if (list[0].score >= gate) federated.push(scope);
+ else alsoMatched.push({ scope, bestId: list[0].id });
+ }
+
+ // An id can only belong to one scope (scope is a path prefix), but keep the
+ // defensive max so a duplicate cannot double-count or flip its attribution.
+ const acc = new Map<string, { scope: string; score: number }>();
+ for (const scope of federated) {
+ for (const d of byScope.get(scope)!) {
+ const prev = acc.get(d.id);
+ if (!prev || d.score > prev.score) acc.set(d.id, { scope, score: d.score });
+ }
+ }
+
+ let max = 0;
+ for (const e of acc.values()) if (e.score > max) max = e.score;
+ const ranked = [...acc.entries()].map(([id, e]) => ({
+ id,
+ scope: e.scope,
+ score: max > 0 ? e.score / max : 0,
+ }));
+ ranked.sort(
+ (a, b) => b.score - a.score || a.scope.localeCompare(b.scope) || a.id.localeCompare(b.id),
+ );
+ return { ranked, federated, alsoMatched };
+}
+
/** The per-scope scoring hooks `rankScopesAndFuse` drives — the caller owns
* the actual lexical math and graph walk (they need its corpus state); this
* module owns the orchestration so the shape of "rank per scope, then fuse"
* lives in one place. */
export interface ScopeRankOps {
- /** Lexical scores for the scope's own docs (positive entries only),
- * computed against the scope's OWN corpus statistics. */
+ /** Lexical scores for the scope's own docs (positive entries only), computed
+ * against the REPO-WIDE corpus statistics. Per-scope statistics would make
+ * the returned numbers incomparable between scopes, which is exactly the
+ * defect in #117 — see the module header. */
lex(scope: string): Map<string, number>;
- /** The match-STRENGTH signal for the participation gate, evaluated at one
- * doc id within `scope` (always called on that scope's raw-lex top doc):
- * `coverage` is the idf-weighted matched share over name+path+body;
- * `coverageStrong` is the same over NAME only, with file-kind nodes
- * contributing zero (see `strongShare` in `src/ask/ask.ts`, the function
- * this must be backed by). Computed against the scope's own per-scope idf,
- * same as `lex`. */
- strength(scope: string, id: string): { coverage: number; coverageStrong: number };
+ /** Does ANY scoring doc in this scope match a query term in its NAME or PATH?
+ *
+ * Serves ONE narrow purpose: suppressing a scope whose entire claim on the
+ * query is prose. Score cannot catch that on its own — BM25 rewards a term
+ * repeated in comments, so a file mentioning "gateway" thirty times in a
+ * banner can outscore the class actually named `GatewayTimeout`.
+ *
+ * Deliberately evaluated over the whole scope, not just its top hit, and as a
+ * presence check rather than a share threshold. Both choices are load-bearing:
+ * a share of the whole query is unreachable for long queries (which is how the
+ * old gate ended up admitting one scope and hiding the rest), and judging a
+ * scope by its single best doc suppressed any scope whose top hit was a file
+ * node — measured at 7.6 points of pooled R@10. */
+ hasIdentifierMatch?(scope: string): boolean;
/** Graph walk restricted to the scope's subgraph, seeded by that scope's
* lexical scores; returns top-normalized centrality (or an empty map). */
walk(scope: string, seeds: Map<string, number>): Map<string, number>;
@@ -187,23 +282,26 @@ export interface ScopeRankOps {
/**
* Multi-scope orchestration for `ask`'s symbol ranking: for each scope, score
- * lexically, gate by match STRENGTH (not a lenient ratio), walk the scope's
- * subgraph, blend exactly like the single-scope path (`lexN +
- * graphWeight·pr`, both per-scope-normalized; walk-rescued nodes join above
- * `rescueFloor`), then fuse the surviving scopes' blended lists with
- * {@link fuseScopes}. A scope with zero lexical matches contributes nothing
- * (no seeds → no walk → no docs).
+ * lexically (against the repo-wide statistics the caller supplies), walk the
+ * scope's subgraph, blend exactly like the single-scope path (`lexN +
+ * graphWeight·pr`, walk-rescued nodes joining above `rescueFloor`), then
+ * combine the scopes with {@link combineComparableScopes}. A scope with zero
+ * lexical matches contributes nothing (no seeds → no walk → no docs).
*
- * The gate is the SAME two-floor signal `federateAsk` (workspace federation,
- * `src/graph/workspace.ts`) uses: a scope participates iff its top hit's
- * `coverageStrong` ≥ {@link STRONG_FLOOR} OR its `coverage` ≥
- * {@link HIGH_FLOOR}. It is evaluated here, on each scope's raw-lex top doc,
- * BEFORE normalization/walk/blend — gated-out scopes never enter the
- * walk/blend/fuse pipeline at all, they go straight to `alsoMatched` with
- * their best raw-lex doc id. (`fuseScopes`'s own internal ratio gate still
- * runs on the survivors' already-normalized input afterward, but since every
- * survivor's blended top is ~1.0 by construction, that gate is now just a
- * no-op safety net rather than the real gate — the real gate is this one.)
+ * The one thing that differs from the single-scope path: `lexN` divides by
+ * `globalMaxLex`, the best raw lexical score across ALL scopes, rather than by
+ * each scope's own best. That shared denominator is what makes the blended
+ * scores comparable, and therefore what lets the combined order be read off the
+ * score. Dividing per scope — as this did before #117 — pinned every scope's
+ * top hit to ~1.0 regardless of how well it matched, which is the defect.
+ *
+ * Participation is then the {@link PARTICIPATION_RATIO} share check inside
+ * {@link combineComparableScopes}: a scope must reach a real fraction of the
+ * best score anywhere in the repo, else it is reported in `alsoMatched` for
+ * `--in`. The match-STRENGTH floors ({@link STRONG_FLOOR}/{@link HIGH_FLOOR})
+ * are no longer applied here — they existed to stand in for a share check that
+ * per-scope normalization had made vacuous. `federateAsk` still normalizes per
+ * child repo, so it still gates on strength; see `src/graph/workspace.ts`.
*/
export function rankScopesAndFuse(
scopes: string[],
@@ -222,60 +320,59 @@ export function rankScopesAndFuse(
const alsoMatched: FusionResult["alsoMatched"] = [];
const scoped: ScopedDoc[] = [];
- // Per-scope raw best + strength, computed once so we can pick survivors and,
- // if the gate would exclude EVERY scope, rescue the strongest one (below).
- const meta = new Map<string, { lex: Map<string, number>; maxLex: number; bestId: string; strength: number; passes: boolean }>();
+ // Per-scope raw best, plus the best raw score anywhere in the repo. That
+ // repo-wide maximum is the shared denominator every scope normalizes by.
+ const meta = new Map<string, { lex: Map<string, number>; maxLex: number; bestId: string }>();
+ let globalMaxLex = 0;
for (const [scope, lex] of lexByScope) {
let maxLex = 0, bestId = "";
for (const [id, v] of lex) if (v > maxLex || (v === maxLex && id < bestId)) { maxLex = v; bestId = id; }
if (maxLex <= 0) continue;
- const { coverage, coverageStrong } = ops.strength(scope, bestId);
- // strength is the scale-INVARIANT [0,1] match share (name vs name+path+body);
- // used to pick the rescue scope fairly across corpora, unlike raw maxLex.
- meta.set(scope, { lex, maxLex, bestId, strength: Math.max(coverage, coverageStrong), passes: coverageStrong >= STRONG_FLOOR || coverage >= HIGH_FLOOR });
+ meta.set(scope, { lex, maxLex, bestId });
+ if (maxLex > globalMaxLex) globalMaxLex = maxLex;
}
- const survivors = [...meta.entries()].filter(([, m]) => m.passes).map(([s]) => s);
- // Anti-abstain rescue: the strength gate is a SHARE of the whole query, so a
- // long/verbose query (a pasted issue body) can push every scope below the
- // floor and make `ask` return nothing — measurably worse than grep. When the
- // gate would exclude every scope yet real matches exist, keep the single
- // strongest scope so `ask` returns best-effort hits (the escalation nudge
- // already tells the agent to switch to grep if they look weak) instead of
- // abstaining. The gate still does its real job — suppressing a weak scope
- // beside a strong one — whenever any scope passes.
- if (survivors.length === 0 && meta.size > 0) {
- // Tiebreak on scale-invariant match STRENGTH (not raw per-corpus maxLex,
- // which would let the biggest scope's numerically-larger score win — the very
- // cross-corpus scale bias RRF exists to avoid). Fall back to maxLex only when
- // strength ties (e.g. all zero), then scope name for determinism.
- survivors.push(
- [...meta.entries()].sort(
- (a, b) => b[1].strength - a[1].strength || b[1].maxLex - a[1].maxLex || a[0].localeCompare(b[0]),
- )[0][0],
- );
+ // Body-only suppression. With a shared denominator a weak scope simply
+ // scores low, so participation is decided on score in
+ // `combineComparableScopes` — no scope is excluded on a share threshold any
+ // more. The single exception is a scope in which NO doc matched a query term
+ // in a name or path while some other scope did: that is the comment-banner
+ // collision (see `hasIdentifierMatch` above), and score cannot separate it
+ // from a real match. Only applied when a genuine identifier match exists
+ // somewhere, so a query that legitimately matches only prose still returns
+ // something.
+ const bodyOnly = new Set<string>();
+ if (ops.hasIdentifierMatch) {
+ const named = new Map<string, boolean>();
+ for (const scope of meta.keys()) named.set(scope, ops.hasIdentifierMatch(scope));
+ if ([...named.values()].some(Boolean)) {
+ for (const [scope, ok] of named) if (!ok) bodyOnly.add(scope);
+ }
+ }
+ for (const scope of bodyOnly) {
+ alsoMatched.push({ scope, bestId: meta.get(scope)!.bestId });
+ meta.delete(scope);
}
- for (const [scope, m] of meta) if (!survivors.includes(scope)) alsoMatched.push({ scope, bestId: m.bestId });
+ // The denominator must come from the scopes that actually rank, or a
+ // suppressed scope could still set the scale for everyone else.
+ globalMaxLex = 0;
+ for (const { maxLex } of meta.values()) if (maxLex > globalMaxLex) globalMaxLex = maxLex;
- for (const scope of survivors) {
- const { lex, maxLex } = meta.get(scope)!;
+ for (const [scope, { lex }] of meta) {
const pr = ops.walk(scope, lex);
const candidates = new Set(lex.keys());
for (const [id, p] of pr) if (p >= rescueFloor) candidates.add(id);
for (const id of candidates) {
- const lexN = maxLex > 0 ? (lex.get(id) ?? 0) / maxLex : 0;
+ const lexN = globalMaxLex > 0 ? (lex.get(id) ?? 0) / globalMaxLex : 0;
const blended = (lexN + graphWeight * (pr.get(id) ?? 0)) * (ops.rankFactor?.(scope, id) ?? 1);
if (blended > 0) scoped.push({ id, scope, score: blended });
}
}
- // fuseScopes's own gate still runs on this (already-survivor, already-
- // normalized) input — left as-is per its pure-test contract; it's now
- // effectively a no-op safety net rather than the real gate.
- const fused = fuseScopes(scoped);
+ const combined = combineComparableScopes(scoped);
return {
- ranked: fused.ranked,
- federated: fused.federated,
- alsoMatched: [...alsoMatched, ...fused.alsoMatched],
+ ranked: combined.ranked,
+ federated: combined.federated,
+ alsoMatched: [...alsoMatched, ...combined.alsoMatched],
};
}
diff --git a/src/graph/workspace.ts b/src/graph/workspace.ts
index 396d0b41..3ef30419 100644
--- a/src/graph/workspace.ts
+++ b/src/graph/workspace.ts
@@ -8,9 +8,13 @@
* { "version": 1, "children": ["repoA", "repoB"] }
*
* Queries run at the parent federate across the children: `ask` fuses every
- * child's ranked hits with the same RRF `fuseScopes` used inside a single
- * multi-scope repo (so the big repo can't drown the small one), `grep`/`map`/
- * `check`/`callers` run per child and merge, always labeled `<child>/`.
+ * child's ranked hits with `fuseScopes`, reciprocal-rank fusion, so the big
+ * repo can't drown the small one. Scopes INSIDE one repo no longer take that
+ * path — they share corpus statistics and a normalization denominator, so they
+ * are combined by score (see `ask/fuse.ts`). Separate child repositories have
+ * no such shared scale, which is exactly the case rank fusion is for.
+ * `grep`/`map`/`check`/`callers` run per child and merge, always labeled
+ * `<child>/`.
*
* This module owns the pure/core pieces (the format, its readers/writers, graph
* loading, and the federated command bodies that return renderable data). The
@@ -188,18 +192,25 @@ interface ChildRun {
// STRONG_FLOOR / HIGH_FLOOR (a child federates if its top hit matched a
// query term in a NAME/PATH field ≥ STRONG_FLOOR, OR its overall name+path+
// body coverage is broad enough to be real even body-only, ≥ HIGH_FLOOR) are
-// defined ONCE in `../ask/fuse.js` and imported here — `rankScopesAndFuse`
-// (single-graph multi-scope, `src/ask/fuse.ts`) gates on the exact same two
-// floors, so the two paths that solve the identical "a weak scope must not
-// federate beside a strong one" problem can never drift apart again.
+// defined ONCE in `../ask/fuse.js` and imported here. This is now their only
+// caller: a share-of-the-query threshold is the best available signal when the
+// candidates carry no common score scale, which is the situation across child
+// repositories. `rankScopesAndFuse` used to gate on them for the same reason
+// and no longer needs to — within one repo the scores are comparable, so a
+// weak scope is held back by scoring low rather than by a strength floor.
/**
- * Federated `ask` across a workspace: run each child's own per-scope ask
- * pipeline, then fuse ALL the children's scope lists with the same
- * `fuseScopes` used inside a single multi-scope repo. Each child contributes
- * one fusion scope per intra-child scope, labeled `<child>/<scope>` (or just
- * `<child>` for a child's root scope), so a big repo can't drown a small one —
- * rank positions are comparable across repos where raw scores are not.
+ * Federated `ask` across a workspace: run each child's own ask pipeline, then
+ * fuse ALL the children's scope lists with `fuseScopes` (reciprocal rank). Each
+ * child contributes one fusion scope per intra-child scope, labeled
+ * `<child>/<scope>` (or just `<child>` for a child's root scope), so a big repo
+ * can't drown a small one — rank positions are comparable across repos where
+ * raw scores are not.
+ *
+ * Rank fusion is still right HERE, and only here: each child was scored against
+ * its own corpus, so cross-child scores share no scale. Within a single repo
+ * that is no longer true — `rankScopesAndFuse` gives every scope the same
+ * statistics and the same denominator and combines them by score.
*
* Returns a normal `AskResult`; `formatAsk` renders it unchanged, labeling each
* hit `[<child>/…]` via the standard multi-scope path.
diff --git a/test/ask-fusion.test.ts b/test/ask-fusion.test.ts
index 30fe57f6..e9d15a6d 100644
--- a/test/ask-fusion.test.ts
+++ b/test/ask-fusion.test.ts
@@ -108,28 +108,29 @@ test("deterministic under input shuffle, including score ties", () => {
}
});
-// ── rankScopesAndFuse: the participation gate is the SAME two-floor
-// match-STRENGTH signal `federateAsk` (workspace federation) uses — a scope
-// participates iff its top hit's `coverageStrong` (name-only, file-kind-
-// excluded idf share) ≥ STRONG_FLOOR OR its `coverage` (name+path+body idf
-// share) ≥ HIGH_FLOOR. Gated-out scopes go to `alsoMatched`. This is NOT the
-// old `PARTICIPATION_RATIO` (0.25 × raw-lexical-best) ratio — Task 5's
-// investigation proved that ratio "far too lenient, leaks junk, worsens as
-// corpus grows" for exactly this reason: a body-only incidental collision's
-// raw lexical score can clear 0.25× of another scope's best even though it
-// never matched a name/path field at all. These drive `rankScopesAndFuse` end
-// to end (through its `lex`/`strength`/`walk` callback seam, the same seam
-// `ask.ts` drives it through) so a regression here is caught even if
-// `fuseScopes`'s own (pure, post-normalization, ratio-based) gate still
-// passes its tests — that gate is now just a no-op safety net once real
-// scopes have already survived this one. ────────────────────────────────────
-
-/** Scope "a": five docs with a real raw-lex spread (best = 10), always a
- * strong (name-matched) hit. Scope "b": one doc whose raw score is fixed but
- * whose match-strength (`coverage`/`coverageStrong`) is the probe knob — the
- * exact signal the real gate checks, decoupled from the raw-lex ratio the old
- * (buggy) gate used. No graph walk (PR rescue isn't what's under test). */
-function makeOps(bStrength: { coverage: number; coverageStrong: number }): ScopeRankOps {
+// ── rankScopesAndFuse: since #117, scopes are scored against REPO-WIDE corpus
+// statistics and normalized by a shared denominator, so their scores are
+// directly comparable. Participation is therefore the PARTICIPATION_RATIO
+// share check — a scope must reach 0.25× the best score anywhere in the repo —
+// and gated-out scopes still go to `alsoMatched` for `--in`.
+//
+// Historical note, because it looks like a reversal: this ratio check WAS
+// previously rejected as "far too lenient, leaks junk", and a match-STRENGTH
+// gate (STRONG_FLOOR/HIGH_FLOOR) replaced it. That judgement was correct at the
+// time and is now obsolete for a specific reason — back then every scope was
+// normalized by its OWN best, so each scope's top doc sat at ~1.0 and the ratio
+// compared 1.0 against 1.0, which is vacuous. Against a shared denominator the
+// same ratio compares real magnitudes and does the job it was designed for. The
+// strength floors are still exported and still gate `federateAsk`, which fuses
+// separate repositories and so genuinely has no shared denominator.
+//
+// These drive `rankScopesAndFuse` end to end through its `lex`/`walk` callback
+// seam — the same seam `ask.ts` drives it through. ─────────────────────────
+
+/** Scope "a": five docs with a real raw-lex spread (best = 10). Scope "b": one
+ * doc whose raw score is the probe knob, since raw magnitude is now what
+ * decides participation. No graph walk (PR rescue isn't what's under test). */
+function makeOps(bBest: number): ScopeRankOps {
const aLex = new Map([
["a1", 10],
["a2", 8],
@@ -137,10 +138,9 @@ function makeOps(bStrength: { coverage: number; coverageStrong: number }): Scope
["a4", 4],
["a5", 2],
]);
- const bLex = new Map([["b1", 3]]); // raw score irrelevant to the new gate
+ const bLex = new Map([["b1", bBest]]);
return {
lex: (s) => (s === "a" ? new Map(aLex) : new Map(bLex)),
- strength: (s) => (s === "a" ? { coverage: 1, coverageStrong: 1 } : bStrength),
walk: () => new Map(),
};
}
@@ -150,7 +150,6 @@ test("rankScopesAndFuse: post-normalization rank factors cannot be erased by a t
// The test remains the strongest raw match even after lexical de-ranking.
// Normalization therefore restores it to 1.0 while source lands at 0.4.
lex: () => new Map([["source", 4], ["test", 10]]),
- strength: () => ({ coverage: 1, coverageStrong: 1 }),
walk: () => new Map(),
rankFactor: (_scope, id) => (id === "test" ? 0.35 : 1),
};
@@ -162,12 +161,12 @@ test("rankScopesAndFuse: post-normalization rank factors cannot be erased by a t
);
});
-test("rankScopesAndFuse: a scope with ONLY a body-token match (coverageStrong 0, coverage below HIGH_FLOOR) is excluded from ranked and reported in alsoMatched", () => {
- // b's top hit never matched a name/path field (coverageStrong = 0) and its
- // overall coverage (0.3) is well under HIGH_FLOOR (0.5) — an incidental
- // body-comment collision, same shape as the monorepo junk/ repro.
- const r = rankScopesAndFuse(["a", "b"], makeOps({ coverage: 0.3, coverageStrong: 0 }), 0.5, 0.05);
- assert.deepEqual(r.federated, ["a"], "b must not federate on a body-only, sub-floor match");
+test("rankScopesAndFuse: a scope whose best hit is a weak incidental match is excluded from ranked and reported in alsoMatched", () => {
+ // b's only hit scores 1 against a repo best of 10 — 0.1, well under the 0.25
+ // participation ratio. An incidental body-comment collision, same shape as
+ // the monorepo junk/ repro.
+ const r = rankScopesAndFuse(["a", "b"], makeOps(1), 0.5, 0.05);
+ assert.deepEqual(r.federated, ["a"], "b must not federate on a weak match");
assert.deepEqual(r.alsoMatched, [{ scope: "b", bestId: "b1" }]);
assert.ok(
r.ranked.every((d) => d.scope === "a"),
@@ -176,29 +175,32 @@ test("rankScopesAndFuse: a scope with ONLY a body-token match (coverageStrong 0,
assert.deepEqual(
r.ranked.map((d) => d.id),
["a1", "a2", "a3", "a4", "a5"],
- "sole surviving scope passes through in raw score order",
+ "sole surviving scope passes through in score order",
);
});
-test("rankScopesAndFuse: positive — a scope with a real NAME/PATH match (coverageStrong ≥ STRONG_FLOOR) federates", () => {
- const r = rankScopesAndFuse(["a", "b"], makeOps({ coverage: 0.2, coverageStrong: STRONG_FLOOR }), 0.5, 0.05);
- assert.deepEqual([...r.federated].sort(), ["a", "b"], "b federates at exactly the strength gate");
+test("rankScopesAndFuse: positive — a scope with a genuinely competitive match federates", () => {
+ const r = rankScopesAndFuse(["a", "b"], makeOps(9), 0.5, 0.05);
+ assert.deepEqual([...r.federated].sort(), ["a", "b"], "b federates on a strong score");
assert.deepEqual(r.alsoMatched, []);
assert.ok(
r.ranked.some((d) => d.id === "b1"),
- "b's doc appears in the fused order",
+ "b's doc appears in the combined order",
);
+ // Comparability: b's 9 sits between a's 10 and a's 8, which is precisely what
+ // per-scope normalization could not express — it put b1 level with a1.
+ assert.deepEqual(r.ranked.slice(0, 3).map((d) => d.id), ["a1", "b1", "a2"]);
});
-test("rankScopesAndFuse: recall valve — body-only (coverageStrong 0) but BROAD (coverage ≥ HIGH_FLOOR) still federates", () => {
- const r = rankScopesAndFuse(["a", "b"], makeOps({ coverage: HIGH_FLOOR, coverageStrong: 0 }), 0.5, 0.05);
- assert.deepEqual([...r.federated].sort(), ["a", "b"], "a broad body-only match still federates via HIGH_FLOOR");
+test("rankScopesAndFuse: a scope exactly at the participation ratio federates (inclusive)", () => {
+ const r = rankScopesAndFuse(["a", "b"], makeOps(10 * PARTICIPATION_RATIO), 0.5, 0.05);
+ assert.deepEqual([...r.federated].sort(), ["a", "b"]);
assert.deepEqual(r.alsoMatched, []);
assert.ok(r.ranked.some((d) => d.id === "b1"));
});
-test("formatAsk: 'also matched … --in …' actually renders for a scope gated out on match strength", () => {
- const fusion = rankScopesAndFuse(["a", "b"], makeOps({ coverage: 0.3, coverageStrong: 0 }), 0.5, 0.05);
+test("formatAsk: 'also matched … --in …' actually renders for a scope gated out of participation", () => {
+ const fusion = rankScopesAndFuse(["a", "b"], makeOps(1), 0.5, 0.05);
const hits: AskHit[] = fusion.ranked.map((d) => ({
kind: "symbol",
title: d.id,
diff --git a/test/ask-scope-comparability.test.ts b/test/ask-scope-comparability.test.ts
new file mode 100644
index 00000000..4e5d5ace
--- /dev/null
+++ b/test/ask-scope-comparability.test.ts
@@ -0,0 +1,168 @@
+/**
+ * Multi-scope score comparability (issue #117).
+ *
+ * The failure these pin: inside ONE repo, each scope used to be scored against
+ * its own corpus statistics and then normalized by its OWN best hit, so every
+ * scope's top document came out at ~1.0 no matter how weakly it actually
+ * matched. Reciprocal-rank fusion then combined those orderings by RANK, which
+ * discards magnitude entirely — a barely-relevant scope's rank-1 file tied the
+ * genuinely-relevant scope's rank-1 file, and pushed that scope's rank-2 file
+ * (frequently the answer) down the list.
+ *
+ * The fix makes scores comparable across scopes — global corpus statistics, one
+ * shared normalization denominator — and then ranks by score. These tests drive
+ * `rankScopesAndFuse` through the same `lex`/`strength`/`walk` seam `ask.ts`
+ * uses, so they exercise the production path rather than a reimplementation.
+ *
+ * Deliberately free of benchmark-specific assumptions: the fixtures are raw
+ * lexical scores, not files, languages, or repository shapes.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { rankScopesAndFuse, PARTICIPATION_RATIO, type ScopeRankOps } from "../src/ask/fuse.js";
+
+/**
+ * Two scopes, both passing the match-strength signal, with a large RAW score
+ * gap between them:
+ *
+ * core/ best 10 — the query genuinely belongs here; `core2` is the answer
+ * samples/ best 3 — a real but far weaker match
+ *
+ * Under per-scope normalization both scopes' rank-1 normalize to 1.0 and fuse
+ * to an identical RRF contribution, so `samples1` lands above `core2`. Under a
+ * shared denominator `samples1` scores 0.3 against `core2`'s 0.8 and the order
+ * is right. No graph walk: PR rescue is not what is under test.
+ */
+function twoScopeOps(samplesBest = 3): ScopeRankOps {
+ const core = new Map([
+ ["core1", 10],
+ ["core2", 8],
+ ["core3", 6],
+ ["core4", 4],
+ ["core5", 2],
+ ]);
+ const samples = new Map([["samples1", samplesBest]]);
+ return {
+ lex: (s) => new Map(s === "core" ? core : samples),
+ // Both scopes clear the strength signal — this test is about magnitude,
+ // not about whether a scope matched a name field.
+ strength: () => ({ coverage: 1, coverageStrong: 1 }),
+ walk: () => new Map(),
+ };
+}
+
+test("#117: a weak scope's best hit must not outrank a strong scope's second hit", () => {
+ const r = rankScopesAndFuse(["core", "samples"], twoScopeOps(), 0.5, 0.05);
+ const order = r.ranked.map((d) => d.id);
+ const core2 = order.indexOf("core2");
+ const weak = order.indexOf("samples1");
+
+ assert.ok(core2 >= 0, `core2 must be ranked at all, got ${order.join(" > ")}`);
+ assert.ok(weak >= 0, `samples1 federates on strength, so it must appear, got ${order.join(" > ")}`);
+ assert.ok(
+ core2 < weak,
+ `core2 (raw 8, strong scope) must outrank samples1 (raw 3, weak scope).\n` +
+ ` got: ${order.join(" > ")}\n` +
+ ` this is #117: per-scope normalization made both scopes' rank-1 equal, ` +
+ `so the weak scope's only hit displaced the strong scope's second hit.`,
+ );
+});
+
+test("#117: cross-scope order follows score magnitude, not per-scope rank", () => {
+ const r = rankScopesAndFuse(["core", "samples"], twoScopeOps(), 0.5, 0.05);
+ const order = r.ranked.map((d) => d.id);
+ assert.deepEqual(
+ order.slice(0, 4),
+ ["core1", "core2", "core3", "core4"],
+ `the strong scope's run must not be interleaved by a weaker scope's rank-1, got ${order.join(" > ")}`,
+ );
+});
+
+test("#117: scores stay comparable across scopes — the weak scope's best scores strictly below the strong scope's best", () => {
+ const r = rankScopesAndFuse(["core", "samples"], twoScopeOps(), 0.5, 0.05);
+ const byId = new Map(r.ranked.map((d) => [d.id, d.score]));
+ assert.ok(
+ byId.get("samples1")! < byId.get("core1")!,
+ `a weaker raw match must carry a strictly lower fused score ` +
+ `(samples1=${byId.get("samples1")}, core1=${byId.get("core1")})`,
+ );
+});
+
+// ── invariants that must survive the change ────────────────────────────────
+
+test("small-scope discoverability: a SMALL but genuinely strong scope still ranks near the top", () => {
+ // The property per-scope ranking exists to protect. `tiny` holds one file and
+ // would be buried by raw pooling; it matches nearly as well as the huge
+ // scope's best, so it must stay visible.
+ const ops: ScopeRankOps = {
+ lex: (s) =>
+ s === "huge"
+ ? new Map([["h1", 10], ["h2", 9], ["h3", 8], ["h4", 7], ["h5", 6]])
+ : new Map([["t1", 9.5]]),
+ strength: () => ({ coverage: 1, coverageStrong: 1 }),
+ walk: () => new Map(),
+ };
+ const r = rankScopesAndFuse(["huge", "tiny"], ops, 0.5, 0.05);
+ const order = r.ranked.map((d) => d.id);
+ assert.ok(
+ order.indexOf("t1") >= 0 && order.indexOf("t1") <= 2,
+ `a one-file scope with a near-best match must stay in the top 3, got ${order.join(" > ")}`,
+ );
+ assert.deepEqual([...r.federated].sort(), ["huge", "tiny"], "both scopes federate");
+});
+
+test("alsoMatched: a scope too weak to federate is excluded from the order and reported for --in", () => {
+ // With comparable scores the PARTICIPATION_RATIO gate is meaningful again:
+ // 1 / 10 = 0.1, below the 0.25 floor. Before comparability every scope's top
+ // normalized to ~1.0, which made this gate vacuous.
+ const r = rankScopesAndFuse(["core", "samples"], twoScopeOps(1), 0.5, 0.05);
+ assert.deepEqual(r.federated, ["core"], "the weak scope must not federate");
+ assert.deepEqual(
+ r.alsoMatched,
+ [{ scope: "samples", bestId: "samples1" }],
+ "it must still be reported so the caller can suggest --in samples/",
+ );
+ assert.ok(
+ r.ranked.every((d) => d.scope === "core"),
+ "a gated scope contributes nothing to the fused order",
+ );
+});
+
+test("a scope exactly at the participation ratio still federates (gate stays inclusive)", () => {
+ const r = rankScopesAndFuse(["core", "samples"], twoScopeOps(10 * PARTICIPATION_RATIO), 0.5, 0.05);
+ assert.deepEqual([...r.federated].sort(), ["core", "samples"]);
+ assert.deepEqual(r.alsoMatched, []);
+});
+
+test("single-scope repos are untouched: raw score order, original scores, no gate", () => {
+ const ops: ScopeRankOps = {
+ lex: () => new Map([["only1", 4], ["only2", 2], ["only3", 0.1]]),
+ strength: () => ({ coverage: 1, coverageStrong: 1 }),
+ walk: () => new Map(),
+ };
+ const r = rankScopesAndFuse(["only"], ops, 0.5, 0.05);
+ assert.deepEqual(r.ranked.map((d) => d.id), ["only1", "only2", "only3"]);
+ assert.deepEqual(r.federated, ["only"]);
+ assert.deepEqual(r.alsoMatched, []);
+ // One scope means its own best IS the shared denominator, so normalization is
+ // unchanged and the top doc still lands at exactly 1.0 after the blend.
+ assert.equal(r.ranked[0].score, 1);
+});
+
+test("post-normalization rank factors still survive (test de-ranking is not erased)", () => {
+ const ops: ScopeRankOps = {
+ lex: () => new Map([["source", 4], ["test", 10]]),
+ strength: () => ({ coverage: 1, coverageStrong: 1 }),
+ walk: () => new Map(),
+ rankFactor: (_scope, id) => (id === "test" ? 0.35 : 1),
+ };
+ const r = rankScopesAndFuse(["app"], ops, 0.5, 0.05);
+ assert.deepEqual(r.ranked.map((d) => d.id), ["source", "test"]);
+});
+
+test("deterministic under scope-order shuffle", () => {
+ const a = rankScopesAndFuse(["core", "samples"], twoScopeOps(), 0.5, 0.05);
+ const b = rankScopesAndFuse(["samples", "core"], twoScopeOps(), 0.5, 0.05);
+ assert.deepEqual(a.ranked, b.ranked, "fused order must not depend on scope iteration order");
+ assert.deepEqual([...a.federated].sort(), [...b.federated].sort());
+});