Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(mcp): lead subsystem questions with their concept page (#1046)
Raghav Chamadiyad6828bc43c6b
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline.pyindex 1a4dab571..bec3fc218 100644--- a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py@@ -437,3 +437,235 @@ async def expand_via_graph(hits: list[dict], ctx: Any) -> list[dict]:combined = hits + additionscombined.sort(key=lambda h: h["score"], reverse=True)return combined+++# ---------------------------------------------------------------------------+# Stage: parent-concept surfacing (subsystem-shaped questions)+# ---------------------------------------------------------------------------++# A subsystem-shaped question asks about a part of the system as a whole:+# "overview of X", "main parts of X", "what subsystem does Y belong to",+# "where would I add a Z". For these the best answer is the concept page that+# documents the whole subsystem, not one of its member files or a more specific+# child concept page. Retrieval ranks the specific children above the parent (a+# child's embedding matches the query's noun more tightly than the broader+# parent), so the parent concept/rollup page never surfaces even though it+# exists. This gate is purely on the query's natural-language shape — no+# repo-specific vocabulary — so it generalises across codebases.+# Kept deliberately high-precision: bare fragments like "parts of", "belongs to",+# "structure of", or "which module" appear constantly inside ordinary how/where+# questions ("what parts of the code touch this", "which module imports config"),+# so each phrase is anchored to an unambiguous subsystem-overview intent. Better+# to miss an oddly-worded subsystem question (it just keeps today's ranking) than+# to reorder an implementation question's file hits.+_SUBSYSTEM_QUERY_RE = re.compile(+ r"\boverview of\b|\bgive me an overview\b|\bhigh[- ]level\b|"+ r"\bmain parts of\b|\bwhat (?:are|is) the (?:main )?(?:parts|pieces|components) of\b|"+ r"\barchitecture of\b|\bwalk me through\b|"+ r"\bhow (?:is|are|does) .+ (?:organi[sz]ed|structured|laid out|put together)\b|"+ r"\bwhat subsystem\b|\bwhich subsystem\b|"+ r"\bwhere would i (?:add|put)\b|\bwhere do i (?:add|put)\b",+ re.IGNORECASE,+)++# How many of the strongest real (non-noise) hits to cluster when looking for+# their shared subsystem. Decision records and pages without a path are skipped+# so a query whose top slots are crowded by decision noise still clusters on its+# real member files.+_PARENT_EXPAND_TOP_N = 8+# A directory is a tight structural cluster when it is the immediate parent of at+# least this many surfaced hits. A lone surfaced file is not enough structural+# signal on its own; that case is left to the semantic concept lookup.+_PARENT_MIN_SHARE = 2+# The injected parent leads the surface (it is the answer) but only just: a+# small multiplier over the current top score keeps it first without+# manufacturing a dominant retrieval that would inflate confidence to "high".+_PARENT_EXPAND_BOOST = 1.02+# How deep to look in a concept-restricted vector search for the subsystem page+# a query is semantically about. The window is wide because concept pages are a+# small minority of the corpus, so the right one can sit below many file/symbol+# hits before this filter drops those away.+_CONCEPT_FETCH_LIMIT = 60+++def is_subsystem_query(question: str) -> bool:+ """True when the question asks about a subsystem/module as a whole, so the+ concept page for that subsystem should lead rather than its member files."""+ return bool(_SUBSYSTEM_QUERY_RE.search(question or ""))+++def _common_ancestor(paths: set[str]) -> str:+ """Longest shared directory prefix of the given paths, segment-wise."""+ split = [p.split("/") for p in paths]+ common: list[str] = []+ for segs in zip(*split, strict=False):+ if len(set(segs)) == 1:+ common.append(segs[0])+ else:+ break+ return "/".join(common)+++async def _semantic_concept_paths(question: str, ctx: Any) -> list[str]:+ """Target paths of the concept/layer pages a concept-restricted vector search+ ranks highest for the question, best-first.++ This is the "guaranteed concept-page candidate": when a subsystem query's+ file hits land in the wrong neighborhood (a UI consumer of the subsystem+ rather than the subsystem itself), the subsystem's own concept page still+ matches the query semantically and surfaces here even though it never+ entered the file-dominated main fetch. Best-effort: returns [] on any error+ so the caller degrades to the structural path.+ """+ vs = getattr(ctx, "vector_store", None)+ if vs is None:+ return []+ try:+ results = await asyncio.wait_for(+ vs.search(question, limit=_CONCEPT_FETCH_LIMIT), timeout=8.0+ )+ except Exception:+ return []+ out: list[str] = []+ for r in results:+ if getattr(r, "page_type", "") in ("module_page", "layer_page"):+ tp = getattr(r, "target_path", "") or ""+ if not tp:+ pid = getattr(r, "page_id", "") or ""+ tp = pid.split(":", 1)[1] if ":" in pid else pid+ if tp and tp not in out:+ out.append(tp)+ return out+++async def expand_via_parent_page(hits: list[dict], question: str, ctx: Any) -> list[dict]:+ """Lead a subsystem-shaped question with the concept page for its subsystem.++ Two complementary signals pick that page, neither tuned to any repository:++ * Structural — when the surfaced file hits cluster under an ancestor+ directory that has a concept/rollup page, that ancestor IS the subsystem.+ Ancestors are found by walking each hit's target_path up the tree (the+ same relationship the generator uses to mint rollups). The tightest+ (deepest) ancestor covering >=2 hits wins, never a catch-all near the root.+ * Semantic — when the file hits land in the wrong neighborhood (a consumer+ of the subsystem, not the subsystem), the subsystem's own concept page+ still matches the query and is recovered by a concept-restricted vector+ search. Used only when no structural cluster is found, so a strong file+ cluster is never overridden by a semantic guess.++ The chosen page is promoted in place if already retrieved (its children+ out-embed it, so the cap drops it) or injected as the leading hit otherwise.+ A no-op on every non-subsystem question, so file/implementation queries keep+ today's ranking untouched.+ """+ if not hits or not is_subsystem_query(question):+ return hits+ # Cluster on the strongest real hits: skip decision records and any hit+ # without a path so decision noise crowding the top slots can't starve the+ # clustering of the member files that reveal the subsystem.+ top = [+ h+ for h in hits+ if h.get("target_path") and h.get("page_type") != "decision_record"+ ][:_PARENT_EXPAND_TOP_N]+ if not top:+ return hits++ # For each surfaced hit, count its immediate parent directory and credit+ # every strict ancestor with covering it. A dir that is the immediate parent+ # of two or more surfaced hits is a TIGHT cluster: the query's own member+ # files sit directly in it. Coverage by a distant ancestor (a broad root that+ # merely contains scattered hits) is deliberately not enough — that is what+ # separates a real subsystem from the repository root.+ imm_count: dict[str, int] = {}+ covers: dict[str, set[str]] = {}+ for h in top:+ tp = h["target_path"].rstrip("/")+ parent = tp.rsplit("/", 1)[0] if "/" in tp else ""+ if parent:+ imm_count[parent] = imm_count.get(parent, 0) + 1+ anc = parent+ while anc:+ covers.setdefault(anc, set()).add(tp)+ anc = anc.rsplit("/", 1)[0] if "/" in anc else ""++ tight_clusters = {d for d, c in imm_count.items() if c >= _PARENT_MIN_SHARE}+ semantic_paths = await _semantic_concept_paths(question, ctx)+ candidates = set(covers) | set(imm_count) | set(semantic_paths)+ if not candidates:+ return hits++ async with get_session(ctx.session_factory) as session:+ rows = (+ await session.execute(+ select(Page.target_path, Page.title, Page.summary, Page.page_type).where(+ Page.target_path.in_(candidates),+ Page.page_type.in_(("module_page", "layer_page")),+ )+ )+ ).all()+ if not rows:+ return hits+ by_path = {r[0]: r for r in rows}++ # A tight structural cluster is the query's own files agreeing on a subsystem,+ # so it outranks the semantic guess. When several sibling dirs each cluster+ # (a subsystem split into subdirectories), roll up to their common ancestor+ # page so the answer is the subsystem, not one arbitrary half of it. With no+ # tight cluster the file hits landed in the wrong neighborhood, so the concept+ # page the query is semantically about wins instead.+ tight_pages = tight_clusters & by_path.keys()+ winner = None+ if tight_pages:+ rollup = _common_ancestor(tight_pages)+ if len(tight_pages) > 1 and rollup in by_path and rollup in covers:+ winner = by_path[rollup]+ else:+ best_tp = max(tight_pages, key=lambda d: (imm_count[d], d.count("/")))+ winner = by_path[best_tp]+ else:+ # No tight cluster: the semantically-closest concept page (best-first),+ # then any weak cover ancestor as a last resort.+ winner = next((by_path[tp] for tp in semantic_paths if tp in by_path), None)+ if winner is None:+ cov_pages = [a for a in covers if a in by_path]+ if cov_pages:+ winner = by_path[max(cov_pages, key=lambda a: (a.count("/"), len(covers[a])))]+ if winner is None:+ return hits++ best_tp, best_title, best_summary, best_pt = winner+ top_score = max((h.get("score", 0.0) for h in hits), default=0.0) or 1.0+ lead_score = top_score * _PARENT_EXPAND_BOOST++ # Already in the candidate set but ranked below its own children (they embed+ # tighter to the query noun than the broader parent), so the top-5 cap drops+ # it. Promote it to lead in place rather than adding a duplicate. The bump is+ # small (just past the top hit) so it never manufactures a dominant retrieval+ # that would read "high".+ for h in hits:+ if h.get("target_path") == best_tp:+ if h.get("score", 0.0) < lead_score:+ h["score"] = lead_score+ src = h.get("_sources")+ if isinstance(src, set):+ src.add("parent_promote")+ hits.sort(key=lambda x: x.get("score", 0.0), reverse=True)+ return hits++ # Not retrieved at all: inject it as the leading hit.+ parent_hit = {+ "page_id": f"{best_pt}:{best_tp}",+ "target_path": best_tp,+ "title": best_title or f"Overview: {best_tp}",+ "summary": best_summary or "",+ "snippet": (best_summary or "")[:200],+ "page_type": best_pt or "module_page",+ "score": lead_score,+ "_sources": {"parent_expand"},+ "_expanded_from": "parent",+ }+ combined = [parent_hit, *hits]+ combined.sort(key=lambda h: h["score"], reverse=True)+ return combineddiff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.pyindex a90a03986..ef3face0f 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py@@ -75,6 +75,9 @@ from repowise.server.mcp_server._answer_pipeline import (from repowise.server.mcp_server._answer_pipeline import (expand_via_graph as _expand_via_graph,)+from repowise.server.mcp_server._answer_pipeline import (+ expand_via_parent_page as _expand_via_parent_page,+)from repowise.server.mcp_server._answer_pipeline import (hybrid_retrieve as _hybrid_retrieve,)@@ -807,6 +810,14 @@ async def get_answer(with contextlib.suppress(Exception):async with get_session(ctx.session_factory) as session:hits = await _expand_via_neighbor_rerank(session, repo_id, hits, question, ctx)+ # Parent-concept surfacing: on a subsystem-shaped question ("overview of X",+ # "what subsystem does Y belong to", "where would I add a Z"), lead with the+ # concept/rollup page that documents the whole subsystem instead of the more+ # specific child pages retrieval ranks above it. Structural + query-shape+ # only; a no-op on every other question, so file/implementation queries keep+ # today's ranking. Runs before the cap so the parent can take a top-5 slot.+ with contextlib.suppress(Exception):+ hits = await _expand_via_parent_page(hits, question, ctx)# Demote retrieval noise (decision records on non-why questions, test file# pages on non-test questions) below real pages before the cap, so it can't# occupy a top-5 slot and feed synthesis. Stable and non-dropping; runs after@@ -1429,17 +1440,20 @@ async def get_answer(retrieval_quality = "weak"if hedged:- # Hedged answers: drop the retrieval payload. The consumer has been- # told to read the source — the symbol-docstring blob that helped- # synthesis doesn't help them, and keeping it in the response bloats- # every follow-up turn's prompt cache.+ # Hedged answers: keep the retrieval payload lean but non-empty. The+ # consumer has been told to read the source, but the ranked hits are+ # exactly what tells it WHICH source — and a flow endpoint or a surfaced+ # subsystem page that only lives in this block would otherwise vanish+ # from the response entirely (it is not in citations, which are drawn+ # from the prose). Lean form (no per-hit key_symbols dump) keeps the+ # prompt-cache cost the empty payload was protecting.payload = {"answer": answer_text,"citations": citations,"confidence": confidence,"retrieval_quality": retrieval_quality,- "fallback_targets": fallback_targets[:3],- "retrieval": [],+ "fallback_targets": fallback_targets[:5],+ "retrieval": _serialize_hits(hits, limit=5, lean_symbols=True),"note": ("Synthesis hedged: the LLM could not ground the question in ""the indexed wiki. Read one of fallback_targets to answer."diff --git a/tests/unit/server/mcp/test_answer_calibration.py b/tests/unit/server/mcp/test_answer_calibration.pyindex 754dbb203..4e51b2679 100644--- a/tests/unit/server/mcp/test_answer_calibration.py+++ b/tests/unit/server/mcp/test_answer_calibration.py@@ -387,7 +387,12 @@ async def test_expanded_hedge_marker_downgrades_through_pipeline(setup_mcp, monkresult = await get_answer("What is the default value of min_count_policy?")assert result["confidence"] == "low"- assert result["retrieval"] == []+ # A hedge now hands the agent the ranked candidates to read instead of an+ # empty block: the whole point of the call is to say WHERE the answer is when+ # the prose could not ground it. The list stays lean (no per-hit docstring+ # dump) so the token cost the empty payload was protecting is preserved.+ assert result["retrieval"], "hedged answer should still surface its ranked hits"+ assert all("path" in h for h in result["retrieval"])@pytest.mark.asynciodiff --git a/tests/unit/server/mcp/test_answer_parent_page.py b/tests/unit/server/mcp/test_answer_parent_page.pynew file mode 100644index 000000000..fae4a6105--- /dev/null+++ b/tests/unit/server/mcp/test_answer_parent_page.py@@ -0,0 +1,210 @@+"""Unit tests for get_answer's subsystem-parent surfacing (_answer_pipeline).++A subsystem-shaped question ("overview of X", "what subsystem does Y belong to",+"where would I add a Z") should lead with the concept page for the whole+subsystem, not its member files or a more specific child. ``expand_via_parent_page``+picks that page from two signals: a tight structural cluster (>=2 surfaced hits+whose immediate parent has a concept page) or, when the file hits scatter, the+concept page a concept-restricted vector search ranks highest. It is a no-op on+non-subsystem questions, so file/implementation queries are untouched.+"""++from __future__ import annotations++from dataclasses import dataclass++import pytest+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.persistence.models import Page, Repository+from repowise.server.mcp_server._answer_pipeline import (+ _common_ancestor,+ expand_via_parent_page,+ is_subsystem_query,+)++_NOW = __import__("datetime").datetime(2024, 1, 1)+++# --------------------------------------------------------------------------- #+# Pure query-shape gate — the generality guard: matches natural-language+# subsystem phrasing, never implementation/flow questions (which carry file+# golds this stage must not touch).+# --------------------------------------------------------------------------- #+ "q",+ [+ "Give me an overview of the ingestion subsystem.",+ "What are the main parts of the auth layer?",+ "What subsystem does the parser belong to?",+ "Where would I add a new payment provider?",+ "Walk me through the rendering pipeline.",+ "What are the main components of the scheduler?",+ "How is the auth layer structured?",+ ],+)+def test_subsystem_shapes_match(q):+ assert is_subsystem_query(q)+++ "q",+ [+ "how does the parser compute a symbol's line range",+ "how does a request flow from the router to the handler",+ "verify_and_heal",+ "why is the cache invalidated on write",+ "what fields does the config blob contain",+ # Bare fragments that appear inside ordinary implementation questions and+ # must NOT trip the subsystem gate (would otherwise reorder file hits).+ "which module handles retries in this function",+ "how is data validated before it belongs to the batch",+ "structure of the JSON payload returned by this endpoint",+ "components of the response are validated where",+ "what parts of the code touch this variable",+ "how does data flow through parts of the pipeline",+ ],+)+def test_implementation_shapes_do_not_match(q):+ assert not is_subsystem_query(q)+++def test_common_ancestor():+ assert _common_ancestor({"a/b/c", "a/b/d"}) == "a/b"+ assert _common_ancestor({"a/b/x", "a/c/y"}) == "a"+ assert _common_ancestor({"a/b"}) == "a/b"+ assert _common_ancestor({"x/y", "p/q"}) == ""+++# --------------------------------------------------------------------------- #+# Fakes + helpers for the DB-backed stage.+# --------------------------------------------------------------------------- #+@dataclass+class _VecResult:+ page_id: str+ page_type: str+ target_path: str+++class _FakeVectorStore:+ """Returns a fixed concept-page ranking regardless of the query."""++ def __init__(self, ranked_paths):+ self._ranked = ranked_paths++ async def search(self, query, limit=10):+ return [+ _VecResult(f"module_page:{p}", "module_page", p) for p in self._ranked[:limit]+ ]+++class _Ctx:+ def __init__(self, factory, vector_store):+ self.session_factory = factory+ self.vector_store = vector_store+++def _hit(page_type, target_path, score, sources=("vector",)):+ return {+ "page_id": f"{page_type}:{target_path}",+ "page_type": page_type,+ "target_path": target_path,+ "score": score,+ "_sources": set(sources),+ }+++async def _seed(session: AsyncSession, module_paths):+ session.add(+ Repository(+ id="r1", name="r", url="u", local_path="/t", default_branch="main",+ settings_json="{}", created_at=_NOW, updated_at=_NOW,+ )+ )+ for p in module_paths:+ session.add(+ Page(+ id=f"module_page:{p}", repository_id="r1", page_type="module_page",+ title=f"{p.rsplit('/', 1)[-1]} overview", content="c", summary="s",+ target_path=p, source_hash="h", model_name="m", provider_name="m",+ generation_level=4, created_at=_NOW, updated_at=_NOW,+ )+ )+ await session.flush()+++# --------------------------------------------------------------------------- #+# Behaviour of the stage.+# --------------------------------------------------------------------------- #+async def test_noop_on_non_subsystem_question(factory):+ async with factory() as s:+ await _seed(s, ["pkg/ingestion"])+ await s.commit()+ ctx = _Ctx(factory, _FakeVectorStore(["pkg/ingestion"]))+ hits = [_hit("file_page", "pkg/ingestion/parser.py", 5.0)]+ out = await expand_via_parent_page(list(hits), "how does the parser work", ctx)+ assert [h["target_path"] for h in out] == ["pkg/ingestion/parser.py"]+++async def test_tight_cluster_injects_parent(factory):+ async with factory() as s:+ await _seed(s, ["pkg/ingestion"])+ await s.commit()+ ctx = _Ctx(factory, _FakeVectorStore(["pkg/somewhere/else"]))+ # Two files share the immediate parent pkg/ingestion -> tight cluster.+ hits = [+ _hit("file_page", "pkg/ingestion/parser.py", 5.0),+ _hit("file_page", "pkg/ingestion/walker.py", 4.9),+ ]+ out = await expand_via_parent_page(list(hits), "overview of the ingestion subsystem", ctx)+ assert out[0]["target_path"] == "pkg/ingestion" # parent leads+ assert out[0]["score"] >= 5.0+++async def test_dispersed_hits_use_semantic_concept_page(factory):+ async with factory() as s:+ await _seed(s, ["pkg/core/generation"])+ await s.commit()+ # File hits scatter across unrelated dirs: no tight cluster. The concept+ # page the query is about is recovered from the (fake) vector ranking.+ ctx = _Ctx(factory, _FakeVectorStore(["pkg/core/generation"]))+ hits = [+ _hit("module_page", "pkg/ui/wiki", 4.0),+ _hit("file_page", "pkg/types/overview.ts", 3.8),+ _hit("file_page", "pkg/api/overview.ts", 3.7),+ ]+ out = await expand_via_parent_page(list(hits), "overview of the wiki generation subsystem", ctx)+ assert out[0]["target_path"] == "pkg/core/generation"+++async def test_sibling_clusters_roll_up_to_common_ancestor(factory):+ async with factory() as s:+ await _seed(s, ["pkg/health", "pkg/health/biomarkers", "pkg/health/complexity"])+ await s.commit()+ ctx = _Ctx(factory, _FakeVectorStore(["pkg/health/biomarkers"]))+ # biomarkers and complexity each cluster (>=2 immediate children); the answer+ # is their common parent subsystem, not one arbitrary half.+ hits = [+ _hit("file_page", "pkg/health/biomarkers/registry.py", 5.0),+ _hit("file_page", "pkg/health/biomarkers/base.py", 4.9),+ _hit("file_page", "pkg/health/complexity/walker.py", 4.8),+ _hit("file_page", "pkg/health/complexity/engine.py", 4.7),+ ]+ out = await expand_via_parent_page(list(hits), "where would I add a health metric", ctx)+ assert out[0]["target_path"] == "pkg/health"+++async def test_present_parent_is_promoted_not_duplicated(factory):+ async with factory() as s:+ await _seed(s, ["pkg/ingestion"])+ await s.commit()+ ctx = _Ctx(factory, _FakeVectorStore(["pkg/ingestion"]))+ # The parent page is already retrieved but ranked below its children.+ hits = [+ _hit("module_page", "pkg/ingestion/parser", 5.0),+ _hit("module_page", "pkg/ingestion/walker", 4.9),+ _hit("module_page", "pkg/ingestion", 2.0),+ ]+ out = await expand_via_parent_page(list(hits), "overview of the ingestion subsystem", ctx)+ assert out[0]["target_path"] == "pkg/ingestion" # promoted to lead+ assert [h["target_path"] for h in out].count("pkg/ingestion") == 1 # no duplicate