Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(answer): name what each candidate file defines, not just its path (#1306)
Raghav Chamadiyab24226c04c71
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.pyindex 25ce6a574..92a37b2cc 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py@@ -152,6 +152,7 @@ from repowise.server.mcp_server.tool_answer.symbols import (_concept_anchor_hits,_extract_question_identifiers,_extract_value_answer,+ _hydrate_candidate_defines,_hydrate_symbols_for_hits,_read_symbol_source,build_homonym_union_bodies,@@ -945,6 +946,15 @@ async def get_answer(await _hydrate_symbols_for_hits(session, repo_id, hits, ctx, question_ids=question_ids)+ # And the shortlist BELOW the synthesis cap: `candidates` names+ # those files and, until now, said nothing about any of them.+ # Runs here, sharing the open session, and against+ # `resolved_pool` rather than `hits` because the whole point is+ # the files the cap discarded. Suppressed with the block above:+ # a missing `_defines` costs a `defines` key, never an answer.+ await _hydrate_candidate_defines(+ session, repo_id, resolved_pool, question_ids=question_ids+ )# --- Qualified-miss guard ----------------------------------------------# The question qualified a symbol (``Parent.leaf``) but the exact-name scandiff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.pyindex 8417e3915..8773465e2 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py@@ -44,6 +44,30 @@ _INLINE_BODY_MAX_SYMBOLS = 2# body of ~99% of symbols in one shot; the rest carry a continuation token._INLINE_BODY_MAX_LINES = 120+# `candidates[].defines` — what each ranked file actually contains.+#+# `candidates` names up to 20 files and, until now, carried nothing about any of+# them beyond an optional line span. Measured on the 25 flow questions, 434 of+# the 499 paths a get_answer response served carried no content at all, and the+# Layer B taxonomy judged 89% of the agent's post-answer searches as EXPAND: it+# took a name we served and went to fetch the substance we did not attach. A+# file named without its definitions is a Grep the agent has to run.+#+# This is bounded hard rather than generously, because the same measurements say+# our payload is ALREADY the larger context (13,958 chars against a bare agent's+# 12,735) and scores lower. The goal is substance per served path, not more+# chars: names and line numbers, no signatures, no docstrings, no bodies.+#+# `_DEFINES_CHAR_BUDGET` is the ceiling for the whole block in one response and+# is spent in retrieval-rank order, so the best-ranked file is the one that+# always gets described. `_DEFINES_PER_CANDIDATE` stops a 200-symbol module from+# consuming the budget before the second candidate is reached.+_DEFINES_PER_CANDIDATE = 6+_DEFINES_CHAR_BUDGET = 1500+# Files whose symbols get looked up at all. Matches _CANDIDATE_LIMIT: querying+# beyond what `candidates` can emit is wasted work.+_DEFINES_MAX_FILES = 20+# Synthesis-body depth for the top question-relevant symbols. The default# _MATCHED_SYMBOL_SOURCE_LINES (40) excerpt cuts a docstring-heavy definition# off before its answer-bearing logic reaches the LLM, so synthesis hedges@@ -328,7 +352,11 @@ _HIGH_CONFIDENCE_SCORE_FLOOR = 1.5# answer-grounding earns high on a non-dominant retrieval. Cached pre-v11 rows# carry the old body-dump / dominance-only grade and must bypass so the# recalibrated confidence reaches callers.-_ANSWER_SCHEMA_VERSION = 11+# v12: `candidates[].defines` — each ranked file now names the definitions it+# contains. Cached pre-v12 rows carry the bare {path, lines} shape, which is the+# named-but-not-carried payload this version exists to replace, so they must+# bypass rather than serve the old block back.+_ANSWER_SCHEMA_VERSION = 12# Hard TTL on answer-cache rows. Commit-based invalidation (the payload's# stamped ``_indexed_commit`` vs the repo's current head) is the primarydiff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.pyindex f0980625d..a8115aaf8 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py@@ -21,6 +21,7 @@ from repowise.server.mcp_server.tool_answer.config import (_BACKEND_PATH_PREFIXES,_BACKEND_QUESTION_TOKENS,_COVERAGE_FLOOR,+ _DEFINES_CHAR_BUDGET,_DOMAIN_PENALTY,_GATED_EXCERPT_CHARS,_PAGE_EXCERPT_HITS,@@ -65,9 +66,24 @@ def serialize_candidates(hits: list[dict], *, limit: int = _CANDIDATE_LIMIT) ->like a path" heuristic says yes and the agent that opens it gets an error.Scoring impact is about zero, measured; it is wrong on the same argumentA14 was, which is that this field has one meaning and it is not "page id".++ ``defines`` carries what the file declares, as ``name:line`` pairs, when+ ``_hydrate_candidate_defines`` resolved any. That is the difference between+ a path the agent has to Grep and a line it can read: 434 of the 499 paths a+ get_answer response served on the 25 flow questions carried no content+ whatsoever, and the Layer B taxonomy judged 89% of the agent's post-answer+ searches to be exactly that expansion. **Line numbers here are as indexed+ and are not verified against the live file** (unlike ``get_symbol``); they+ are navigation, not a citation.++ **Which paths are emitted, and in what order, is not affected by any of+ this.** ``defines`` is attached to entries the existing loop already built,+ so an added or exhausted budget can never add, drop or reorder a path."""out: list[dict] = []seen: set[str] = set()+ # Spent in rank order, so the best-ranked file is the one always described.+ budget = _DEFINES_CHAR_BUDGETfor h in hits:path = hit_file_path(h)if not path or path in seen:@@ -79,6 +95,11 @@ def serialize_candidates(hits: list[dict], *, limit: int = _CANDIDATE_LIMIT) ->ends = [s["end_line"] for s in symbols if s.get("end_line")]if starts and ends:entry["lines"] = f"{min(starts)}-{max(ends)}"+ if budget > 0 and h.get("_defines"):+ defines = ", ".join(f"{name}:{line}" for name, line in h["_defines"])+ if len(defines) <= budget:+ entry["defines"] = defines+ budget -= len(defines)out.append(entry)if len(out) >= limit:breakdiff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.pyindex fda4ab963..c498eb4ed 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/symbols.py@@ -13,8 +13,11 @@ from typing import Anyfrom sqlalchemy import selectfrom repowise.core.persistence.models import WikiSymbol+from repowise.server.mcp_server._page_paths import hit_file_pathfrom repowise.server.mcp_server._verify import verify_and_healfrom repowise.server.mcp_server.tool_answer.config import (+ _DEFINES_MAX_FILES,+ _DEFINES_PER_CANDIDATE,_ENRICH_TOP_N_HITS,_HIGH_CONFIDENCE_SCORE_FLOOR,_HOMONYM_UNION_BODY_MAX_LINES,@@ -574,6 +577,111 @@ async def _concept_anchor_hits(return hits+# Definition kinds worth naming in `defines`, best first. A file is+# characterised by what it declares, so a class outranks a bare function and+# both outrank a variable. Kinds absent from this map are not emitted at all:+# imports and re-exports would fill the budget with names that answer nothing.+_DEFINE_KIND_RANK = {+ "class": 0,+ "interface": 1,+ "struct": 1,+ "enum": 1,+ "type": 2,+ "function": 3,+ "method": 4,+ "constant": 5,+}+++async def _hydrate_candidate_defines(+ session,+ repo_id: str,+ hits: list[dict],+ question_ids: set[str] | None = None,+) -> None:+ """Mutate *hits* in place: attach ``_defines`` to the candidate-pool files.++ ``candidates`` names the files retrieval ranked and, before this, said+ nothing about any of them. An agent handed ``django/shortcuts.py`` and+ nothing else has exactly one move available, which is to go and Grep it; the+ Layer B taxonomy judged 89% of post-answer searches to be that move. Naming+ the definitions a file contains turns "search this file" into "read this+ line", and often answers a where-is-it question outright.++ Deliberately cheap and deliberately shallow:++ * **One batched query**, on ``(repository_id, file_path)`` which the+ ``uq_wiki_symbol`` index already covers, over at most+ ``_DEFINES_MAX_FILES`` paths. No live file reads, no bounds verification.+ * **Names and start lines only.** No signature, no docstring, no body. Those+ already have homes (``retrieval[].key_symbols``, ``symbol_bodies``) and+ this block must not compete with them for the payload's byte budget.+ * **Line numbers are index-recorded, not verified.** Unlike ``get_symbol``,+ nothing here checks the stored bounds against the live file. They are a+ navigation hint; the serializer's field documentation says so.++ Ordering within a file: question-named symbols first (they are what the+ agent came for), then by declaration kind, then by position. Dunders and+ private names are dropped unless the question named them.+ """+ qids = {q.lower() for q in (question_ids or set())}++ paths: list[str] = []+ seen: set[str] = set()+ for h in hits:+ p = hit_file_path(h)+ if not p or p in seen:+ continue+ seen.add(p)+ paths.append(p)+ if len(paths) >= _DEFINES_MAX_FILES:+ break+ if not paths:+ return++ res = await session.execute(+ select(+ WikiSymbol.file_path,+ WikiSymbol.name,+ WikiSymbol.kind,+ WikiSymbol.start_line,+ ).where(+ WikiSymbol.repository_id == repo_id,+ WikiSymbol.file_path.in_(paths),+ )+ )++ by_file: dict[str, list[tuple[int, int, str, int]]] = {}+ for file_path, name, kind, start_line in res.all():+ rank = _DEFINE_KIND_RANK.get((kind or "").lower())+ if rank is None or not name:+ continue+ matched = name.lower() in qids+ if not matched and name.startswith("_"):+ continue+ by_file.setdefault(file_path, []).append(+ (0 if matched else 1, rank, name, start_line or 0)+ )++ for path, rows in by_file.items():+ rows.sort(key=lambda r: (r[0], r[1], r[3]))+ picked: list[tuple[str, int]] = []+ taken: set[str] = set()+ for _m, _r, name, start in rows:+ if name in taken:+ continue+ taken.add(name)+ picked.append((name, start))+ if len(picked) >= _DEFINES_PER_CANDIDATE:+ break+ by_file[path] = picked # type: ignore[assignment]++ for h in hits:+ p = hit_file_path(h)+ if p and by_file.get(p):+ h["_defines"] = by_file[p]++async def _hydrate_symbols_for_hits(session,repo_id: str,diff --git a/tests/unit/server/mcp/test_answer_candidate_defines.py b/tests/unit/server/mcp/test_answer_candidate_defines.pynew file mode 100644index 000000000..84425d0af--- /dev/null+++ b/tests/unit/server/mcp/test_answer_candidate_defines.py@@ -0,0 +1,129 @@+"""``_hydrate_candidate_defines``: what each ranked file declares, and in what order.++`candidates` names up to 20 files. Until this, it said nothing about any of+them, so an agent handed `django/shortcuts.py` had exactly one move available+and that move was a Grep. Measured on the 25 flow questions: 434 of the 499+paths a get_answer response served carried no content at all, and the Layer B+taxonomy judged 89% of the agent's post-answer searches to be exactly that+expansion from a bare name.++The selection rules are what make this cheap enough to ship at 6 names per file:+question-named symbols first (they are what the agent came for), then by+declaration kind, then by position, with imports and private names dropped.+"""++from __future__ import annotations++import pytest++from repowise.core.persistence.models import WikiSymbol+from repowise.server.mcp_server.tool_answer.config import _DEFINES_PER_CANDIDATE+from repowise.server.mcp_server.tool_answer.symbols import _hydrate_candidate_defines+++def _sym(repo_id, n, *, path, name, kind, line):+ return WikiSymbol(+ id=f"d{n}",+ repository_id=repo_id,+ file_path=path,+ symbol_id=f"{path}::{name}",+ name=name,+ qualified_name=name,+ kind=kind,+ signature="",+ start_line=line,+ end_line=line + 5,+ )+++async def shortcuts(session, repo_id):+ rows = [+ _sym(repo_id, 1, path="shortcuts.py", name="redirect", kind="function", line=20),+ _sym(repo_id, 2, path="shortcuts.py", name="render", kind="function", line=24),+ _sym(repo_id, 3, path="shortcuts.py", name="resolve_url", kind="function", line=146),+ _sym(repo_id, 4, path="shortcuts.py", name="Http404", kind="class", line=8),+ _sym(repo_id, 5, path="shortcuts.py", name="_helper", kind="function", line=200),+ _sym(repo_id, 6, path="shortcuts.py", name="loader", kind="import", line=1),+ _sym(repo_id, 7, path="other.py", name="Elsewhere", kind="class", line=3),+ ]+ for r in rows:+ session.add(r)+ await session.commit()+ return rows+++async def test_a_class_outranks_a_function_and_position_breaks_the_tie(+ session, repo_id, shortcuts+) -> None:+ hits = [{"target_path": "shortcuts.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert hits[0]["_defines"] == [+ ("Http404", 8),+ ("redirect", 20),+ ("render", 24),+ ("resolve_url", 146),+ ]+++async def test_a_question_named_symbol_is_promoted_to_the_front(+ session, repo_id, shortcuts+) -> None:+ """The agent that asked about `resolve_url` should not read past three+ other names to find it -- promotion is the whole reason this is+ question-aware rather than a static file outline."""+ hits = [{"target_path": "shortcuts.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits, question_ids={"resolve_url"})+ assert hits[0]["_defines"][0] == ("resolve_url", 146)+++async def test_imports_are_not_offered_as_definitions(session, repo_id, shortcuts) -> None:+ """An import fills the budget with a name that answers nothing: the symbol+ it names is defined in a file we are not describing."""+ hits = [{"target_path": "shortcuts.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert "loader" not in [n for n, _ in hits[0]["_defines"]]+++async def test_a_private_name_is_dropped_unless_the_question_asked_for_it(+ session, repo_id, shortcuts+) -> None:+ hits = [{"target_path": "shortcuts.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert "_helper" not in [n for n, _ in hits[0]["_defines"]]++ hits = [{"target_path": "shortcuts.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits, question_ids={"_helper"})+ assert hits[0]["_defines"][0] == ("_helper", 200)+++async def test_symbols_do_not_leak_across_files(session, repo_id, shortcuts) -> None:+ hits = [{"target_path": "shortcuts.py"}, {"target_path": "other.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert [n for n, _ in hits[1]["_defines"]] == ["Elsewhere"]+++async def test_a_dense_file_cannot_consume_the_whole_block(session, repo_id) -> None:+ for i in range(40):+ session.add(_sym(repo_id, 100 + i, path="dense.py", name=f"f{i}", kind="function", line=i))+ await session.commit()+ hits = [{"target_path": "dense.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert len(hits[0]["_defines"]) == _DEFINES_PER_CANDIDATE+++async def test_a_file_with_no_indexed_symbols_gets_no_key_at_all(session, repo_id) -> None:+ """Absent, not empty. `serialize_candidates` tests `h.get("_defines")`, and+ an empty list would emit `"defines": ""` into every payload."""+ hits = [{"target_path": "undocumented.py"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert "_defines" not in hits[0]+++async def test_a_symbol_page_is_resolved_to_its_file(session, repo_id, shortcuts) -> None:+ """`shortcuts.py::resolve_url` is a page id. The symbol rows are keyed on+ the file path, so a spotlight hit must be resolved before the lookup or it+ silently returns nothing."""+ hits = [{"target_path": "shortcuts.py::resolve_url"}]+ await _hydrate_candidate_defines(session, repo_id, hits)+ assert hits[0].get("_defines"), "a symbol-page hit must still describe its file"diff --git a/tests/unit/server/mcp/test_answer_candidates.py b/tests/unit/server/mcp/test_answer_candidates.pyindex 5c050a313..3d3b1d7fe 100644--- a/tests/unit/server/mcp/test_answer_candidates.py+++ b/tests/unit/server/mcp/test_answer_candidates.py@@ -92,3 +92,70 @@ def test_an_unhydrated_hit_is_kept_rather_than_dropped():would lose a real file over a join that did not land."""assert serialize_candidates([_hit("a.py")]) == [{"path": "a.py"}]+++# --- `defines`: what each named file actually contains ----------------------+#+# Measured on the 25 flow questions: a get_answer response served 499 paths and+# 65 of them carried any content at all. The Layer B taxonomy judged 89% of the+# agent's post-answer searches as EXPAND, taking a name we served and going to+# fetch the substance we did not attach. `defines` is that substance, at names+# and line numbers only.+++def _hit_d(path, defines, symbols=None):+ h = _hit(path, symbols)+ h["_defines"] = defines+ return h+++def test_defines_names_what_the_file_declares():+ out = serialize_candidates([_hit_d("shortcuts.py", [("resolve_url", 146), ("redirect", 20)])])+ assert out == [{"path": "shortcuts.py", "defines": "resolve_url:146, redirect:20"}]+++def test_a_hit_without_defines_is_unchanged():+ """The pre-change shape survives exactly where nothing was hydrated."""+ assert serialize_candidates([_hit("a.py")]) == [{"path": "a.py"}]+++def test_defines_never_changes_which_paths_are_served_or_their_order():+ """HARD CONSTRAINT: Layer A Coverage scores WHICH files come back.++ Two gate fixes (#1284, #1289) bought +0.216 File Coverage and 10-of-10+ ceiling recovery on the dev half, in this same tool. `defines` is attached+ to entries the existing loop already built, so it cannot add, drop or+ reorder a path -- and this asserts that on the same pool twice rather than+ trusting the argument. An end-to-end version of the same assertion runs over+ 1,392 real candidate paths in+ `50-results/payload-substance/scripts/assert_same_retrieval.py`.+ """+ bare = [_hit("a.py", [{"start_line": 3, "end_line": 9}]), _hit("b.py"), _hit("a.py::X")]+ enriched = [+ _hit_d("a.py", [("f", 3)], [{"start_line": 3, "end_line": 9}]),+ _hit_d("b.py", [("g", 1), ("h", 2)]),+ _hit_d("a.py::X", [("X", 3)]),+ ]+ got = serialize_candidates(enriched)+ stripped = [{k: v for k, v in e.items() if k != "defines"} for e in got]+ assert stripped == serialize_candidates(bare)+++def test_the_char_budget_is_spent_in_rank_order_and_truncates_nothing():+ """Size discipline, and the failure mode it must not have.++ Our payload is already the larger context (13,958 chars against a bare+ agent's 12,735) and scores lower, so this block is hard-capped. The cap+ drops whole `defines` values off the tail, best-ranked file first served; it+ never truncates one mid-string, because a half-written `name:li` is a symbol+ that does not exist. And a dropped `defines` must still leave its path.+ """+ big = [("s" * 40 + str(i), i) for i in range(6)]+ out = serialize_candidates([_hit_d(f"f{i}.py", big) for i in range(30)], limit=20)++ assert [e["path"] for e in out] == [f"f{i}.py" for i in range(20)]+ assert out[0].get("defines"), "the best-ranked candidate must always be described"+ assert not out[-1].get("defines"), "the budget must run out before the tail"+ for e in out:+ if e.get("defines"):+ assert all(":" in pair for pair in e["defines"].split(", "))