Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(why): rank rationale comments by weighted coverage and serve a card's best commits (#3724)

Swati Ahujaf825c1f024e2
diff --git a/packages/server/src/repowise/server/mcp_server/_code_rationale.py b/packages/server/src/repowise/server/mcp_server/_code_rationale.py
index 91ff11c05..34dc44a9b 100644
--- a/packages/server/src/repowise/server/mcp_server/_code_rationale.py
+++ b/packages/server/src/repowise/server/mcp_server/_code_rationale.py
@@ -44,7 +44,14 @@ from repowise.core.analysis.decisions.rationale_comments import (
harvest_file_rationale,
)
from repowise.core.exclusion import build_exclude_spec, is_excluded
-from repowise.server.mcp_server._why_relevance import floor_ranked
+from repowise.server.mcp_server._why_relevance import (
+ clears_floor,
+ floor_ranked,
+ question_terms,
+ relevance,
+ term_idf,
+ word_set,
+)
_log = logging.getLogger("repowise.mcp.code_rationale")
@@ -125,6 +132,11 @@ _MAX_RESULTS = 6
# score. Relative, not absolute: scores scale with how many terms a question has.
_RELEVANCE_FLOOR_FRACTION = 0.5
_NEAR_LINE_WINDOW = 60
+# Added to a block's weighted coverage (0 to 1). A marker says the block is a
+# reason and ranks it over an equal-coverage description; a near line is the
+# symbol get_answer anchored.
+_MARKER_BONUS = 0.25
+_NEAR_LINE_BONUS = 0.25
def _content_terms(query: str | None) -> set[str]:
@@ -143,7 +155,8 @@ def _content_terms(query: str | None) -> set[str]:
# A literal number the question pins down (a cap / limit / count / version).
# ``_content_terms`` drops it (len<3, non-alpha), but the number is often the
# WHOLE point - "why is X capped at 40" vs "...at 600" can only be told apart by
-# the digit. So it is scored separately, and heavily.
+# the digit. So it is matched separately, as a whole token, and weighed by rarity
+# like any term: "40" on one comment decides, "3" on every "Tier 3" does not.
_NUMBER_RE = re.compile(r"(?<![\w.])-?\d+(?:\.\d+)?(?!\w)")
@@ -166,44 +179,91 @@ def _number_in_text(num: str, low: str) -> bool:
return re.search(rf"(?<![\w.]){re.escape(num)}(?![\w.])", low) is not None
+def _number_term(num: str) -> str:
+ """A number's key among the question terms; no word can collide with it."""
+ return f"#{num}"
+
+
def _score_block(
- comment: str, terms: set[str], numbers: list[str]
-) -> tuple[float, list[str], list[str], bool]:
- """Score a comment block. Returns (score, matched_terms, matched_nums, has_marker).
-
- Rationale marker = 2.0; each distinct query term present = 1.0; each literal
- question number present = 3.0 (the strongest single signal - it identifies
- the exact constant the question asks about, cutting through cap/limit
- boilerplate). Uses the broad ``RATIONALE_MARKERS`` set (intent + causal)
- shared with the index-time harvest - recall mode, so intent-only markers
- ("never", "always") count.
+ words: frozenset[str], own: frozenset[str], low: str, idf: dict[str, float]
+) -> tuple[float, list[str], bool] | None:
+ """Score a comment block, or ``None`` when it is not served.
+
+ Returns (score, matched_terms, has_marker). The score is the question's
+ weighted coverage (rarer terms weigh more, over every block scanned) plus
+ the bonuses above. Uses the broad ``RATIONALE_MARKERS`` set shared with the
+ index-time harvest, in recall mode, so intent-only markers ("never",
+ "always") count.
+
+ The gate: with question terms, a marked block needs one of their words in
+ its own text (*own*; *words* adds what an anchoring declaration lends) and
+ an unmarked block also needs the floor's share of their weight; a number
+ alone never qualifies, so "Strategy 3" does not answer "tier 3". Without
+ terms (path mode, "why is this file shaped this way") a marker is enough.
"""
- low = comment.lower()
has_marker = any(m in low for m in RATIONALE_MARKERS)
- matched = sorted(t for t in terms if t in low)
- matched_nums = [n for n in numbers if _number_in_text(n, low)]
- score = (2.0 if has_marker else 0.0) + float(len(matched)) + 3.0 * len(matched_nums)
- return score, matched, matched_nums, has_marker
-
-
-def _keep(
- matched: list[str],
- matched_nums: list[str],
- has_marker: bool,
- has_terms: bool,
-) -> bool:
- """Surfacing gate. A pinned question number co-occurring with a marker or
- any content term is kept (the number disambiguates the exact constant).
- With query terms: need a marker+term overlap, or a strong (>=2 terms)
- overlap on its own. Without query terms (path-mode "why is this file shaped
- this way"): a rationale marker is enough."""
- if matched_nums and (has_marker or matched):
- return True
- if not has_terms:
- return has_marker
- if has_marker and matched:
+ if not idf:
+ return (_MARKER_BONUS, [], True) if has_marker else None
+ matched = [t for t in idf if t in words]
+ coverage = relevance(words, idf)
+ if not any(not t.startswith("#") and t in own for t in matched):
+ return None
+ if not (has_marker or clears_floor(coverage)):
+ return None
+ score = coverage + (_MARKER_BONUS if has_marker else 0.0)
+ return score, sorted(t.lstrip("#") for t in matched), has_marker
+
+
+_IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+
+def _query_identifiers(query: str | None) -> set[str]:
+ """The code names in *query*, lowercased: tokens that split into parts
+ (``_narrow_overload``, ``withManagedSignal``) or lead with an underscore."""
+ return {
+ raw.lower()
+ for raw in _IDENT_RE.findall(query or "")
+ if raw.startswith("_") or len(word_set(raw)) > 1
+ }
+
+
+# A line that declares a name: a keyword declaration, or ``NAME =`` /
+# ``NAME:`` at its start. A call site naming the identifier is not one.
+_DECL_KEYWORD_RE = re.compile(
+ r"\s*(?:(?:export|public|private|protected|internal|static|async|final|pub|"
+ r"readonly|abstract|default|override|open|inline)\s+)*"
+ r"(?:def|class|fn|func|function|const|let|var|val|type|interface|struct|enum)\b"
+)
+_DECL_ASSIGN_RE = re.compile(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?::|=(?!=))")
+
+
+def _declares(line: str, idents: set[str]) -> bool:
+ """Whether *line* declares one of *idents*."""
+ m = _DECL_ASSIGN_RE.match(line)
+ if m and m.group(1).lower() in idents:
return True
- return len(matched) >= 2
+ return bool(_DECL_KEYWORD_RE.match(line)) and bool(
+ idents & {w.lower() for w in _IDENT_RE.findall(line)}
+ )
+
+
+def _anchored_words(lines: list[str], start: int, end: int, idents: set[str]) -> frozenset[str]:
+ """Words of the declaration a block documents, when it declares a query identifier.
+
+ A docstring sits under its ``def``, a comment above its declaration, and
+ neither repeats the name, so the question's strongest term never occurs in
+ the block itself. The line before *start* or the first non-blank line after
+ *end* (1-based) declaring one of *idents* lends the block its words.
+ """
+ if not idents:
+ return frozenset()
+ near = [lines[start - 2]] if start >= 2 else []
+ near += [ln for ln in lines[end : end + 3] if ln.strip()][:1]
+ out: frozenset[str] = frozenset()
+ for ln in near:
+ if _declares(ln, idents):
+ out |= word_set(ln)
+ return out
# A mined row may stand in for the answer only on this many distinct matches
@@ -240,6 +300,30 @@ def _truncate_block(comment: str) -> str:
return comment
+def _file_blocks(
+ root: Path, path: str, idents: set[str], numbers: list[str]
+) -> list[tuple[str, int, int, str, frozenset[str], frozenset[str]]]:
+ """``(path, start, end, comment, words, own words)`` per comment block of *path*."""
+ ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+ text = _read_text(root, path)
+ if text is None or text.count("\n") > _MAX_FILE_LINES:
+ return []
+ try:
+ found = extract_comment_blocks(text, ext, include_trailing=True)
+ except Exception as exc: # never let a tokenizer bug break a tool
+ _log.debug("comment extraction failed for %s: %s", path, exc)
+ return []
+ lines = text.splitlines()
+ out = []
+ for block in found:
+ start, end, comment = block.start_line, block.end_line, block.text
+ low = comment.lower()
+ own = word_set(comment) | {_number_term(n) for n in numbers if _number_in_text(n, low)}
+ lent = _anchored_words(lines, start, end, idents)
+ out.append((path, start, end, comment, own | lent, own))
+ return out
+
+
def mine_rationale(
repo_root: Any,
file_paths: list[str],
@@ -275,76 +359,53 @@ def mine_rationale(
except Exception:
return []
- terms = _content_terms(query)
- numbers = _salient_numbers(query)
- has_terms = bool(terms)
+ terms = question_terms(query, prose=True) if query else []
+ numbers = _salient_numbers(query) if terms else []
+ idents = _query_identifiers(query)
near_lines = near_lines or {}
# Dedupe while preserving order; cap the file fan-out.
- seen: set[str] = set()
- ordered: list[str] = []
- for p in file_paths:
- if p and p not in seen:
- seen.add(p)
- ordered.append(p)
- ordered = ordered[:max_files]
-
- scored: list[tuple[float, bool, bool, dict]] = []
- for path in ordered:
- ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
- text = _read_text(root, path)
- if text is None or text.count("\n") > _MAX_FILE_LINES:
+ ordered = list(dict.fromkeys(p for p in file_paths if p))[:max_files]
+ # Every block of every scanned file first: term rarity is taken over all of
+ # them, so a word most comments here share weighs little.
+ blocks = [b for path in ordered for b in _file_blocks(root, path, idents, numbers)]
+ terms += [_number_term(n) for n in numbers]
+ idf = term_idf(terms, [b[4] for b in blocks]) if terms else {}
+
+ scored: list[tuple[float, dict]] = []
+ for path, start, end, comment, words, own in blocks:
+ result = _score_block(words, own, comment.lower(), idf)
+ if result is None:
continue
+ score, matched, _ = result
near = near_lines.get(path)
- try:
- blocks = extract_comment_blocks(text, ext, include_trailing=True)
- except Exception as exc: # never let a tokenizer bug break a tool
- _log.debug("comment extraction failed for %s: %s", path, exc)
- continue
- for block in blocks:
- start, end, comment = block.start_line, block.end_line, block.text
- score, matched, matched_nums, has_marker = _score_block(comment, terms, numbers)
- if not _keep(matched, matched_nums, has_marker, has_terms):
- continue
- if near is not None and abs(start - near) <= _NEAR_LINE_WINDOW:
- score += 1.5
- # Coalesced runs longer than the cap are split at the head — the
- # lead lines carry the rationale; tail is usually elaboration.
- if end - start + 1 > _MAX_BLOCK_LINES:
- end = start + _MAX_BLOCK_LINES - 1
- scored.append(
- (
- score,
- has_marker,
- bool(matched_nums),
- {
- "path": path,
- "lines": [start, end],
- "comment": _truncate_block(comment) if truncate_blocks else comment,
- "matched_terms": matched + matched_nums,
- },
- )
+ if near is not None and abs(start - near) <= _NEAR_LINE_WINDOW:
+ score += _NEAR_LINE_BONUS
+ # Coalesced runs longer than the cap are split at the head: the
+ # lead lines carry the rationale; tail is usually elaboration.
+ if end - start + 1 > _MAX_BLOCK_LINES:
+ end = start + _MAX_BLOCK_LINES - 1
+ scored.append(
+ (
+ score,
+ {
+ "path": path,
+ "lines": [start, end],
+ "comment": _truncate_block(comment) if truncate_blocks else comment,
+ "matched_terms": matched,
+ },
)
-
- # Precision: a comment with an explicit rationale marker IS the "why". When
- # any survive, drop the marker-less blocks that only cleared the >=2-term
- # gate — on a query with generic terms (lines / source / one) those are
- # usually plain docstrings that read as noise next to the real rationale.
- # The term-only blocks remain the recall fallback when nothing has a marker;
- # a block pinned by a literal question number is always kept (it identifies
- # the exact constant, marker or not).
- if any(m for _, m, _, _ in scored):
- scored = [t for t in scored if t[1] or t[2]]
+ )
scored.sort(key=lambda t: t[0], reverse=True)
# Relevance floor, relative to the best block: below half the top score a
- # block cleared the >=2-term gate on generic vocabulary rather than on the
+ # block cleared the gate on generic vocabulary rather than on the
# question — an SVG edge-routing docstring on "edges"/"project"/"between".
if scored:
floor = scored[0][0] * _RELEVANCE_FLOOR_FRACTION
scored = [t for t in scored if t[0] >= floor]
- return [entry for _, _, _, entry in scored[:max_results]]
+ return [entry for _, entry in scored[:max_results]]
# Cap / limit vocabulary: a question may say "capped" while the source comment
diff --git a/packages/server/src/repowise/server/mcp_server/_query_terms.py b/packages/server/src/repowise/server/mcp_server/_query_terms.py
index a9e368973..1fbfb49c9 100644
--- a/packages/server/src/repowise/server/mcp_server/_query_terms.py
+++ b/packages/server/src/repowise/server/mcp_server/_query_terms.py
@@ -125,7 +125,9 @@ def split_humps(text: str) -> str:
return re.sub(r"(?<=[A-Z])(?=[A-Z][a-z])", " ", out)
-def content_terms(question: str, *, max_terms: int = _MAX_TERMS) -> list[str]:
+def content_terms(
+ question: str, *, max_terms: int | None = _MAX_TERMS, identifiers: bool = True
+) -> list[str]:
"""Content words of *question*, deduplicated, longest first.
Deliberately inclusive: plain lowercase English words are kept, unlike the
@@ -135,11 +137,14 @@ def content_terms(question: str, *, max_terms: int = _MAX_TERMS) -> list[str]:
A camelCase or snake_case token contributes its parts as well as itself, so
someone who writes ``persist_symbols`` and someone who writes "persist
- symbols" reach the same rows.
+ symbols" reach the same rows. With ``identifiers=False`` only the parts
+ are kept: for text written in prose, which names ``persist_symbols`` by its
+ words if at all. ``max_terms=None`` keeps every term.
"""
seen: dict[str, None] = {}
for raw in _WORD_RE.findall(question or ""):
- for tok in (raw, *re.split(r"[^A-Za-z0-9]+", split_humps(raw))):
+ parts = re.split(r"[^A-Za-z0-9]+", split_humps(raw))
+ for tok in (raw, *parts) if identifiers else parts:
t = tok.lower()
if len(t) < _MIN_TERM_LEN or t in STOPWORDS:
continue
diff --git a/packages/server/src/repowise/server/mcp_server/_why_relevance.py b/packages/server/src/repowise/server/mcp_server/_why_relevance.py
index 982468465..f07855c66 100644
--- a/packages/server/src/repowise/server/mcp_server/_why_relevance.py
+++ b/packages/server/src/repowise/server/mcp_server/_why_relevance.py
@@ -18,10 +18,11 @@ anything.
from __future__ import annotations
import math
-from collections.abc import Callable
+import re
+from collections.abc import Callable, Collection, Container
from typing import Any
-from repowise.server.mcp_server._query_terms import content_terms
+from repowise.server.mcp_server._query_terms import content_terms, split_humps
#: Weighted share of a question's vocabulary a record must carry to be served.
#:
@@ -40,14 +41,32 @@ from repowise.server.mcp_server._query_terms import content_terms
RELEVANCE_FLOOR = 0.6
-def question_terms(query: str) -> list[str]:
- """Content words of *query*, singularised and deduplicated, longest first."""
+def question_terms(query: str, *, prose: bool = False) -> list[str]:
+ """Content words of *query*, singularised and deduplicated, longest first.
+
+ ``prose=True`` is for text that names code by its words, comments and
+ commit messages: an identifier counts by its parts and every term is kept.
+ No commit subject spells ``_narrow_overload``, so weighting it as an unseen
+ word put the floor out of reach of every commit for any question naming code.
+ """
seen: dict[str, None] = {}
- for t in content_terms(query):
+ terms = (
+ content_terms(query, max_terms=None, identifiers=False) if prose else content_terms(query)
+ )
+ for t in terms:
seen.setdefault(stem(t), None)
return list(seen)
+def word_set(text: str) -> frozenset[str]:
+ """The singularised words of *text*, for whole-word presence.
+
+ Substring presence lets ``tie`` match "tier" and "entire". Splitting humps
+ and underscores first lets ``withManagedSignal`` carry "managed".
+ """
+ return frozenset(stem(w) for w in re.split(r"[^a-z0-9]+", split_humps(text).lower()) if w)
+
+
def stem(term: str) -> str:
"""``comments`` -> ``comment``. A trailing plural and nothing more.
@@ -62,8 +81,10 @@ def stem(term: str) -> str:
the safe direction and under-stripping is not: presence is a substring test,
so a record writing ``comments`` still contains ``comment``, and ``status``
reduced to ``statu`` still matches both ``status`` and ``statuses``. Words
- ending ``-ies`` are left whole for the same reason — turning ``queries``
+ ending ``-ies`` are left whole for the same reason: turning ``queries``
into ``query`` would stop it matching a record that writes ``queries``.
+ :func:`word_set` applies it to the text as well, so whole-word presence
+ sees both sides through the same rule.
Ceiling: English regular plurals. The upgrade path is a real stemmer, which
is a dependency this earns only once misses appear that a trailing ``s``
@@ -82,7 +103,7 @@ def stem(term: str) -> str:
return term
-def term_idf(terms: list[str], corpus: list[str]) -> dict[str, float]:
+def term_idf(terms: list[str], corpus: Collection[Container[str]]) -> dict[str, float]:
"""``{term: weight}``, rarer terms weighing more, from *corpus* itself.
A count of distinct terms present is the shape the sibling episode ranker
@@ -118,6 +139,8 @@ def term_idf(terms: list[str], corpus: list[str]) -> dict[str, float]:
unseen, outweighed both words that identify the answer. Capping at the
rarest weight the store could actually award says what absence knows without
claiming more than the corpus can support.
+
+ A corpus entry is a string (substring presence) or a :func:`word_set`.
"""
n = len(corpus)
idf: dict[str, float] = {}
@@ -127,7 +150,7 @@ def term_idf(terms: list[str], corpus: list[str]) -> dict[str, float]:
return idf
-def relevance(text: str, idf: dict[str, float]) -> float:
+def relevance(text: Container[str], idf: dict[str, float]) -> float:
"""Share of the question's *weight* that *text* carries, 0.0 to 1.0.
Distinct terms only: a record repeating one word forty times is not more
diff --git a/packages/server/src/repowise/server/mcp_server/tool_why/search.py b/packages/server/src/repowise/server/mcp_server/tool_why/search.py
index 995fcc1a2..7152a02e8 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_why/search.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_why/search.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import contextlib
import json
+from collections.abc import Callable
from typing import Any
from repowise.core.analysis.decision_semantic_match import DECISION_VECTOR_PREFIX
@@ -18,9 +19,13 @@ from repowise.server.mcp_server._why_evidence import annotate_response_evidence_
from repowise.server.mcp_server._why_relevance import (
clears_floor,
query_scorer,
+ question_terms,
recovery_note,
redirect_for,
redirect_when_served,
+ relevance,
+ term_idf,
+ word_set,
)
from repowise.server.mcp_server.tool_why.archaeology import question_commits
from repowise.server.mcp_server.tool_why.basis import _has_archaeology
@@ -109,47 +114,112 @@ def _focus_target_context(
recall = f"get_why(targets={json.dumps(targets)})"
# Term weights from the whole decision corpus: a card's few rows are too
# small a sample for ``term_idf`` (see its warning).
- score = query_scorer(query, corpus)
+ record_score = query_scorer(query, corpus)
+
+ def relevant(row: Any) -> bool:
+ return clears_floor(record_score(json.dumps(row, default=str)))
+
for entry in target_context.values():
if not isinstance(entry, dict):
continue
# ``governing_decisions`` is exempt: an accepted rule binds whatever the
# question. A candidate binds nothing, so it must earn its place.
- _keep_relevant(entry, "candidate_decisions", score, recall)
+ _keep_relevant(entry, "candidate_decisions", relevant, recall)
# Only the origin's decision lane; the author and first commit are
# cheap and wanted whatever the question.
origin = entry.get("origin")
if isinstance(origin, dict):
- _keep_relevant(origin, "linked_decisions", score, recall)
+ _keep_relevant(origin, "linked_decisions", relevant, recall)
arch = entry.get("git_archaeology")
if isinstance(arch, dict):
- _focus_archaeology(arch, score, recall)
+ _focus_archaeology(arch, query, recall)
-def _focus_archaeology(
- arch: dict[str, Any], score: Any, recall: str
-) -> None:
- """Keep the commits that carry the question's terms, count the rest.
+_COMMIT_LANES = ("file_commits", "cross_references", "git_log")
+# A commit carrying one question word shares a word with the question, no more.
+_MIN_COMMIT_TERMS = 2
+# The served commits must beat every other commit by this factor; a near tie
+# says the question does not single any commit out, so the card serves none.
+_COMMIT_MARGIN = 1.25
+# Ties are equal sums of the same weights, up to float rounding.
+_TIE_EPSILON = 1e-9
+
+
+def _focus_archaeology(arch: dict[str, Any], query: str, recall: str) -> None:
+ """Keep the card's best commits for *query*, count the rest.
Here no decision cleared the floor, so these commits are the best evidence
- left.
+ left. Scored on their messages by the question's prose terms, rarity over
+ this card's commits; not the decision corpus, whose rarity is zero for every
+ term on a store with no records.
+
+ Nor the decision floor: a subject is one line, and the commits that answered
+ measured questions carried 0.18 to 0.48 of their weight, so the floor served
+ none. A card serves its strongest commits instead, ties kept, when they
+ carry at least two of the question's terms and clear the margin.
"""
- for lane in ("file_commits", "cross_references", "git_log"):
- _keep_relevant(arch, lane, score, recall)
+ # One entry per commit: the indexed lane and the live log repeat commits,
+ # abbreviating the sha to different lengths (seven is git's own).
+ words: dict[str, frozenset[str]] = {}
+ for lane in _COMMIT_LANES:
+ for r in arch.get(lane) or []:
+ if isinstance(r, dict):
+ words.setdefault(_short_sha(r), _message_words(r))
+ idf = term_idf(question_terms(query, prose=True), list(words.values()))
+ score = {sha: relevance(w, idf) for sha, w in words.items()}
+ eligible = {
+ sha: s for sha, s in score.items() if sum(t in words[sha] for t in idf) >= _MIN_COMMIT_TERMS
+ }
+ best = max(eligible.values(), default=0.0)
+ serve = {sha for sha, s in eligible.items() if best > 0 and s >= best - _TIE_EPSILON}
+ runner_up = max((s for sha, s in score.items() if sha not in serve), default=0.0)
+ if best < _COMMIT_MARGIN * runner_up:
+ serve = set()
+
+ served: set[str] = set()
+ for lane in _COMMIT_LANES:
+ rows = arch.get(lane)
+ if not isinstance(rows, list) or not rows:
+ continue
+ kept, dropped = [], 0
+ for r in rows:
+ sha = _short_sha(r) if isinstance(r, dict) else None
+ if sha in serve and sha not in served:
+ served.add(sha)
+ kept.append(r)
+ elif sha not in serve:
+ dropped += 1
+ # else: served in an earlier lane; not a suppressed commit.
+ if kept:
+ arch[lane] = kept
+ else:
+ arch.pop(lane)
+ if dropped:
+ arch[f"{lane}_omitted"] = recovery_note(dropped, recall)
+
+
+def _short_sha(row: dict) -> str:
+ return str(row.get("sha") or "")[:7]
-def _keep_relevant(block: dict[str, Any], key: str, score: Any, recall: str) -> None:
- """Keep the rows of ``block[key]`` that clear the floor for *score*.
+def _message_words(row: dict) -> frozenset[str]:
+ return word_set(str(row.get("message") or ""))
+
+
+def _keep_relevant(
+ block: dict[str, Any], key: str, relevant: Callable[[Any], bool], recall: str
+) -> None:
+ """Keep the rows of ``block[key]`` that are ``relevant``.
- When none do, the lane becomes ``<key>_omitted``: a count and the recall.
+ When none are, the lane becomes ``<key>_omitted``: a count and the recall.
"""
rows = block.get(key)
if not isinstance(rows, list) or not rows:
return
- kept = [r for r in rows if clears_floor(score(json.dumps(r, default=str)))]
+ kept = [r for r in rows if relevant(r)]
if kept:
block[key] = kept
else:
diff --git a/tests/unit/server/mcp/test_code_rationale.py b/tests/unit/server/mcp/test_code_rationale.py
index 89828b914..694edc47d 100644
--- a/tests/unit/server/mcp/test_code_rationale.py
+++ b/tests/unit/server/mcp/test_code_rationale.py
@@ -87,7 +87,7 @@ def test_mine_ranks_marker_plus_query_overlap_first(tmp_path):
top = out[0]
assert "PYTHONIOENCODING" in top["comment"]
assert top["lines"] == [3, 4]
- assert "windows" in top["matched_terms"]
+ assert "window" in top["matched_terms"]
def test_mine_path_mode_returns_marker_comments_without_query(tmp_path):
@@ -139,23 +139,106 @@ def test_mine_is_bounded_and_safe(tmp_path):
assert mine_rationale(str(tmp_path), [], "x") == []
-def test_mine_drops_markerless_docstrings_when_a_marker_block_wins(tmp_path):
- """Generic query terms (lines/source/one) drag in plain docstrings; once a
- real rationale-marker block is found, the marker-less ones are dropped."""
+def test_mine_ranks_the_marker_block_over_a_generic_docstring(tmp_path):
+ """Generic query terms (lines/source/one) drag in plain docstrings; the
+ marked block that carries the question ranks first, and an unmarked one
+ carrying too little of its weight is dropped."""
src = (
'"""Slice text to source lines; one indexed symbol per call."""\n'
"MAX = 1\n"
"# The source line cap is 600 because round-trip count, not payload\n"
"# size, dominates agent token cost.\n"
"LIMIT = 600\n"
+ '"""Read the source of a file."""\n'
)
p = tmp_path / "s.py"
p.write_text(src, encoding="utf-8")
out = mine_rationale(str(tmp_path), ["s.py"], "why source lines cap one")
assert out, "expected the rationale block to survive"
- # The marker block (has 'because') is the only one kept.
- assert all("because" in r["comment"] for r in out)
- assert any("round-trip count" in r["comment"] for r in out)
+ assert "round-trip count" in out[0]["comment"]
+ assert all("Read the source" not in r["comment"] for r in out)
+
+
+def test_mine_keeps_an_unmarked_block_carrying_the_question_beside_marked_ones(tmp_path):
+ """A docstring stating the behaviour asked about has no causal connective;
+ it must not be dropped because some other block in the file has one."""
+ src = (
+ "# Tiers run in order because the earlier tier is cheaper.\n"
+ "TIERS = 3\n"
+ "def narrow(rep, call):\n"
+ ' """A tie between overloads keeps the representative."""\n'
+ " return rep\n"
+ )
+ _write(tmp_path, "r.py", src)
+ out = mine_rationale(str(tmp_path), ["r.py"], "why does an overload tie keep the representative")
+ assert out and "keeps the representative" in out[0]["comment"]
+
+
+def test_mine_matches_whole_words_not_substrings(tmp_path):
+ """``tie`` is not in "tier" or "entire"."""
+ src = (
+ "# The entire tier is skipped because it is slow.\n"
+ "X = 1\n"
+ "# A tie is broken by name because the order must be stable.\n"
+ "Y = 2\n"
+ )
+ _write(tmp_path, "t.py", src)
+ out = mine_rationale(str(tmp_path), ["t.py"], "why is a tie broken")
+ assert [r["lines"][0] for r in out] == [3]
+
+
+def test_mine_lone_number_does_not_qualify_a_block(tmp_path):
+ """A number counts beside a content term, never on its own."""
+ src = (
+ "# Strategy 3 runs last because it is the most expensive.\n"
+ "X = 1\n"
+ "# Java is left out because its bare-call pattern also matches fields.\n"
+ "LANGS = ()\n"
+ )
+ _write(tmp_path, "n.py", src)
+ out = mine_rationale(str(tmp_path), ["n.py"], "why is java excluded after tier 3")
+ assert [r["lines"][0] for r in out] == [3]
+
+
+def test_mine_anchors_a_block_to_the_symbol_the_query_names(tmp_path):
+ """The comment above a constant never repeats its name; naming it in the
+ question ranks that comment over one sharing only the prose words."""
+ src = (
+ "# Checked in order because java is the common case.\n"
+ "ORDER = ()\n"
+ "\n"
+ "# Kotlin is here and java is not because its receivers differ.\n"
+ "_INHERITED_LANGUAGES = frozenset()\n"
+ )
+ _write(tmp_path, "a.py", src)
+ out = mine_rationale(str(tmp_path), ["a.py"], "why is java not in _INHERITED_LANGUAGES")
+ assert out[0]["lines"][0] == 4
+
+
+def test_mine_does_not_anchor_a_comment_to_a_call_site(tmp_path):
+ """A line that only calls the named symbol lends nothing: the comment below
+ it is about something else, and carries none of the question's words."""
+ src = (
+ "r = _narrow_overload(1)\n"
+ "# never retry here, it is unsafe\n"
+ "x = 2\n"
+ )
+ _write(tmp_path, "c.py", src)
+ out = mine_rationale(str(tmp_path), ["c.py"], "why does _narrow_overload narrow first")
+ assert out == []
+
+
+def test_mine_anchored_block_still_needs_a_word_of_its_own(tmp_path):
+ """The declaration lends its words, but a marked block must carry one
+ question word itself."""
+ src = (
+ "# never retry here, it is unsafe\n"
+ "def _narrow_overload(rep):\n"
+ " return rep\n"
+ )
+ _write(tmp_path, "d.py", src)
+ out = mine_rationale(str(tmp_path), ["d.py"], "why does _narrow_overload narrow first")
+ assert out == []
def test_mine_keeps_term_only_blocks_when_no_marker_exists(tmp_path):
diff --git a/tests/unit/server/mcp/test_why_relevance_floor.py b/tests/unit/server/mcp/test_why_relevance_floor.py
index cbeb8472d..26bab8806 100644
--- a/tests/unit/server/mcp/test_why_relevance_floor.py
+++ b/tests/unit/server/mcp/test_why_relevance_floor.py
@@ -327,3 +327,104 @@ async def get_why_search(query: str, targets: list[str] | None = None) -> dict:
from repowise.server.mcp_server import get_why
return await get_why(query, targets=targets)
+
+
+# --- Commit lanes on a target card -----------------------------------------
+
+
+def _commit(sha: str, message: str) -> dict:
+ return {"sha": sha, "message": message}
+
+
+def test_prose_terms_name_an_identifier_by_its_parts():
+ """No commit subject spells a private helper, so the compound is not a term."""
+ from repowise.server.mcp_server._why_relevance import question_terms, word_set
+
+ terms = question_terms("why does argsMinusFirstX skip _flag_values", prose=True)
+
+ assert "argsminusfirstx" not in terms and "_flag_values" not in terms
+ assert {"arg", "minu", "first", "skip", "flag", "value"} <= set(terms)
+ assert "tie" not in word_set("the entire tier")
+ assert "managed" in word_set("withManagedSignal")
+
+
+def test_a_question_naming_code_keeps_the_commit_that_answers_it():
+ """The decision floor served no commit for a question naming code, and none
+ at all on a store without records. The card's best commit is served."""
+ from repowise.server.mcp_server.tool_why.search import _focus_archaeology
+
+ arch = {
+ "file_commits": [
+ _commit("aaaaaaa1", "fix: don't remove flag value that matches subcommand name"),
+ _commit("bbbbbbb2", "docs: update README"),
+ ],
+ "git_log": [_commit("ccccccc3", "chore: bump flag parser")],
+ }
+ _focus_archaeology(arch, "why does argsMinusFirstX skip the value of a flag", "recall")
+
+ assert [c["sha"] for c in arch["file_commits"]] == ["aaaaaaa1"]
+ assert arch["git_log_omitted"]["suppressed"] == 1
+
+
+def test_a_card_whose_commits_share_one_word_serves_none():
+ from repowise.server.mcp_server.tool_why.search import _focus_archaeology
+
+ arch = {"git_log": [_commit("aaaaaaa1", "fix flag parsing"), _commit("b2", "add tests")]}
+ _focus_archaeology(arch, "why is the mousetrap splash shown for a flag", "recall")
+
+ assert "git_log" not in arch
+ assert arch["git_log_omitted"] == {
+ "suppressed": 2,
+ "reason": "query-irrelevant",
+ "recover_with": "recall",
+ }
+
+
+def test_a_commit_in_two_lanes_is_served_once():
+ """The indexed lane and the live log abbreviate one sha to different lengths."""
+ from repowise.server.mcp_server.tool_why.search import _focus_archaeology
+
+ message = "Allow running persistent run hooks of all parents"
+ arch = {
+ "file_commits": [_commit("4cafa37a", message)],
+ "git_log": [_commit("4cafa37a1b2c", message)],
+ }
+ _focus_archaeology(arch, "why do persistent hooks of all parents run", "recall")
+
+ assert [c["sha"] for c in arch["file_commits"]] == ["4cafa37a"]
+ # Not suppressed: the commit was served from the other lane.
+ assert "git_log" not in arch and "git_log_omitted" not in arch
+
+
+def test_a_near_tie_between_commits_serves_none():
+ """Two commits each carrying two question words at almost the same weight:
+ the question does not single either out."""
+ from repowise.server.mcp_server.tool_why.search import _focus_archaeology
+
+ arch = {
+ "git_log": [
+ _commit("aaaaaaa1", "Delay fetch in tests"),
+ _commit("bbbbbbb2", "Retry body before reading"),
+ _commit("ccccccc3", "Retry before reading"),
+ _commit("ddddddd4", "docs: update README"),
+ ]
+ }
+ _focus_archaeology(arch, "why delay the retry before the fetch body", "recall")
+
+ assert "git_log" not in arch
+ assert arch["git_log_omitted"]["suppressed"] == 4
+
+
+def test_tied_best_commits_are_both_served():
+ from repowise.server.mcp_server.tool_why.search import _focus_archaeology
+
+ arch = {
+ "git_log": [
+ _commit("aaaaaaa1", "Make detection for test-binary more universal"),
+ _commit("bbbbbbb2", 'Revert "Make detection for test-binary more universal"'),
+ _commit("ccccccc3", "docs: update README"),
+ ]
+ }
+ _focus_archaeology(arch, "why is the test binary detected by name", "recall")
+
+ assert [c["sha"] for c in arch["git_log"]] == ["aaaaaaa1", "bbbbbbb2"]