Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(answer): answer retrieval pipeline as a stage package; split _structured_search (#4010)
Raghav Chamadiyaa09e107127c7
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline.pydeleted file mode 100644index 2130dc808..000000000--- a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py+++ /dev/null@@ -1,1251 +0,0 @@-"""Retrieval pipeline for ``get_answer``.--This module owns everything that turns a developer question into a ranked-list of candidate wiki hits — but emphatically not the LLM step, the cache,-or the response shape (those live in ``tool_answer``). Separation of concerns-lets us iterate on retrieval quality without rereading the orchestrator and-vice versa.--Pipeline (each stage is a pure function over hit dicts):-- 1. ``hybrid_retrieve`` : FTS, vector store and the structural symbol- index in parallel, merged via Reciprocal- Rank Fusion. Single retrieval modes- systematically miss either token matches- (vectors drift) or conceptual matches (FTS- is literal), and both page-shaped modes miss- anything a generated file page does not- spell out: its public-symbol table is all- they can index, so a private helper or a- local name is invisible to them. The symbol- leg covers that third class, for the cost of- one more coroutine.- 2. ``hydrate_hits`` — attach target_path, summary, page_type from- the Page table to each hit.- 3. ``apply_pagerank_bias`` — multiply scores by a damped PageRank factor- so architecturally central files outrank- peripheral ones at similar retrieval score.- This is what rescues "how does X work"- questions from peripheral consumers of X.- 4. ``expand_via_graph`` — for top-N hits, walk 1 hop through the graph- and pull in neighbors that have a wiki- page. Rescues near-misses where retrieval- landed in the right module but on a wrong- file (consumer vs. orchestrator). Reference- and co-change edges both qualify; see the- note on the queries for why.--Stages downstream of this module (term coverage, intersection boost, domain-penalty) live in ``tool_answer`` for now — they're tightly coupled to the-existing question-aware symbol promotion code and not worth duplicating.-"""--from __future__ import annotations--import asyncio-import contextlib-import contextvars-import logging-import re-from collections import OrderedDict-from typing import Any--from sqlalchemy import select--from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX-from repowise.core.ingestion.models import CONTAINMENT_EDGE_TYPES-from repowise.core.persistence.database import get_session-from repowise.core.persistence.models import GraphEdge, GraphNode, Page-from repowise.core.providers.embedding import store_has_semantic_vectors-from repowise.core.test_paths import is_test_path-from repowise.server.mcp_server._graph_files import (- is_symbol_node,- keep_projected_edge,- node_to_file,- per_index,-)-from repowise.server.mcp_server._helpers import (- _EMBED_TIMEOUT_ENV,- _VECTOR_TIMEOUT_ENV,- embed_timeout_s,- vector_search_timeout_s,-)-from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES, PAGELESS_FILE, pageless_path-from repowise.server.mcp_server._prose_symbols import filename_backed_pages, symbol_backed_pages--_log = logging.getLogger("repowise.mcp.answer")--# How many candidates each retriever fetches before merging. Both modes-# tend to put the right answer in their top ~10, so 15 gives RRF room to-# resolve ties without dragging weak tail hits into the merge.-_RETRIEVAL_FETCH_LIMIT = 15--# Page-id namespace decision vectors live under in the shared page store. Read-# from the module that writes them so the two cannot drift apart.-_DECISION_PREFIX = DECISION_VECTOR_PREFIX--# Decision records share the page vector store under this page-id namespace, so-# a page fetch has to ask for more rows than it needs and drop them. Over-fetch-# rather than post-filter a fixed window: a query whose nearest neighbours are-# all decisions would otherwise return a candidate set short by however many-# happened to rank high, which reads downstream as thin retrieval rather than as-# a filtered one. The margin is generous because decision rows cluster on the-# question-shaped text that also retrieves well.-_DECISION_OVERFETCH = 15--# RRF constant. The standard k=60 from the original RRF paper — large enough-# that rank-1 (1/61) and rank-2 (1/62) are close, small enough that rank-10-# (1/70) still contributes meaningfully when a hit only shows up in one mode.-_RRF_K = 60--# Multiplier applied to RRF scores so they land in roughly the same numeric-# range as the BM25 raw scores the downstream gates/thresholds (dominance-# ratio, high-confidence score floor, absolute-gap branch) were tuned for.-#-# Top-1 RRF with both modes hitting at rank 0 is 1/60 + 1/60 ≈ 0.033.-# Scaling by 180 puts it at ~6, matching the upper end of observed BM25-# scores on this corpus. Pure ordering preservation — never changes-# *which* hit ranks where, only the absolute numbers the gates compare.-_RRF_SCORE_SCALE = 180.0--# Cap how many extra files graph expansion can add. Without a cap, a hub-# file (many importers) would flood the candidate set and dilute the LLM's-# context budget on tangential neighbors.-_GRAPH_EXPAND_TOP_N = 2-_GRAPH_EXPAND_MAX_NEW = 3--# Degree above which a neighbour is treated as a hub and dropped from the-# expansion set. A file that half the repo imports is a near-neighbour of-# everything, so it says nothing about *this* question; it is also exactly the-# file PageRank ranks first, which is why the ordering below needs a guard-# rather than trusting centrality on its own. Floor plus percentile, so a small-# repo where every file has a handful of edges excludes nothing.-_HUB_DEGREE_FLOOR = 50-_HUB_DEGREE_PERCENTILE = 0.99--# PageRank bias is multiplicative and capped. We don't want a marginally-# more central file to outrank a strong text match — only to break ties.-# Empirically PageRank values on this corpus span ~0 to ~0.01; we normalise-# to the max in the candidate set and scale to a [1.0, 1.3] multiplier.-_PAGERANK_BIAS_MAX = 0.3--# Damping factor for graph-expanded hits. They didn't surface in retrieval,-# so we trust them less than direct hits — but enough to outrank the bottom-# of the top-5 if the parent was strong. 0.7 keeps a strong parent's child-# (e.g. parent at 4.5, expanded child at 3.15) competitive with a real-# rank-3/4 hit (~3.0-3.5).-_GRAPH_EXPAND_DAMPING = 0.7--# Budget for embedding the question is resolved live by embed_timeout_s()-# (see _helpers.py) — REPOWISE_EMBED_TIMEOUT_S, falling back to the 8s a warm-# hosted endpoint needs. The searches that used to embed inline were bounded-# at that same 8s including the embed, so the round-trip keeps that ceiling-# now that it happens on its own.--# Which retrieval legs actually ran for the current question (finding A18).-#-# Every leg here is best-effort by design: a slow vector store must not be able-# to block an answer. The defect was that the fallback was *silent*. An embed-# that times out returns no vector, retrieval quietly continues lexical-only,-# and every health signal still reports green — ``embedder_live`` included,-# because a configured embedder is live whether or not this particular call-# beat the clock. Five queries in one bake-off run were answered without their-# vector leg and nothing in the response, the logs' structured fields, or the-# per-cell record said so.-#-# So the leg outcome travels with the answer. ``embedder_live`` says the-# embedder exists; this says whether it was used. Those are different claims-# and only one of them is about the answer the caller is holding.-_LEG_RECORD: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar(- "repowise_retrieval_legs", default=None-)---def _record_leg(leg: str, outcome: str) -> None:- """Note how one retrieval leg ended, if anyone is collecting."""- record = _LEG_RECORD.get()- if record is not None:- record[leg] = outcome---def begin_leg_record() -> dict[str, str]:- """Start collecting leg outcomes for one question. Returns the record.-- The legs run under ``asyncio.gather``, which copies the context into each- task. Copying rebinds names, not objects, so the tasks all mutate this one- dict and the caller sees what they wrote.- """- record: dict[str, str] = {}- _LEG_RECORD.set(record)- return record---def retrieval_legs() -> dict[str, str]:- """How each retrieval leg ended for the question just answered.-- ``{"fts": "ok", "vector": "timeout", "symbol": "ok"}`` and so on, plus an- ``embed`` key when embedding the question is what failed. Empty before any- retrieval has run.- """- return dict(_LEG_RECORD.get() or {})---def degraded_legs(legs: dict[str, str]) -> list[str]:- """The legs that *broke*, named. Empty when retrieval was whole.-- ``keyless`` is not degradation and is deliberately excluded. A keyless index- has no semantic vectors by construction, so its vector leg is permanently- absent rather than transiently broken, and naming it here would put a- failure marker on every answer the mode ever produces. The configuration- itself is reported once per response by ``_meta`` instead.- """- return sorted(leg for leg, outcome in legs.items() if outcome not in ("ok", "keyless"))--# Third retrieval leg: the structural symbol index. FTS and the vector store-# both read the generated wiki page, which by construction carries an overview-# sentence, the *public* symbol table and dependency paths, with no function-# bodies, no private helpers, no local names. A question about something a page-# does not spell out therefore has nothing to match on, however well it is-# phrased. The symbol index does carry those names, and it is keyed on the-# words a symbol is built from, so it recovers exactly that class of miss.-#-# How many symbols the leg ranks, and how many distinct files they may-# contribute. Deliberately smaller than the other two legs' 15: this leg is-# there to add pool members the page legs cannot see, not to outvote them.-_SYMBOL_LEG_FETCH_LIMIT = 20-_SYMBOL_LEG_MAX_PAGES = 8--# The symbol leg's own RRF constant, larger than the page legs' so its-# contribution is smaller at every rank. Fusing it at k=60 like the others gave-# one lexical symbol match the same weight as a page two independent retrievers-# ranked first, which is not what a name match is worth: measured on the-# 99-question eval it pushed the correct ``distill/skeleton.py`` out of the-# served five in favour of a same-named React component. At k=180 the leg can-# lift a file the page retrievers already liked and can add one they never saw,-# but it cannot outvote them.-_SYMBOL_LEG_RRF_K = 180--# The filename leg is a name match too, weighted like the symbol leg. A name the-# question spells in full (``full_cover``, see ``_prose_symbols``) is fused at the-# page legs' k.-_FILENAME_LEG_RRF_K = _SYMBOL_LEG_RRF_K---# ----------------------------------------------------------------------------# The question's embedding, computed once per call-# -----------------------------------------------------------------------------# Answering one question used to embed its text up to three times: once for the-# main fetch, once for the concept lookup on a subsystem-shaped question, once-# for the neighbourhood re-rank on a flow-shaped one. Each is a network-# round-trip to the embedding provider, billed, for a vector that cannot differ-# between them.-#-# Keyed by ``(id(store), question)`` and holding the store in the value, so an-# id can never be recycled onto a different store while its entries are live.-# Capacity is a handful of entries rather than one: concurrent calls with-# different questions would otherwise evict each other and re-embed. An entry is-# a pure function of (embedder, text), so a hit across calls is as correct as a-# hit within one.-_QUESTION_VECTOR_CACHE_MAX = 4-_QUESTION_VECTORS: OrderedDict[tuple[int, str], tuple[Any, list[float]]] = OrderedDict()---async def question_vector(ctx: Any, question: str) -> list[float] | None:- """The question's embedding, computed at most once per store and question.-- Returns None when there is no store, the backend holds no embedder of its- own, or the embed failed — callers then fall back to the store's own- ``search(text)``, which embeds inline. That fallback is a cost regression,- never a correctness one, so it warns rather than raising.- """- store = getattr(ctx, "vector_store", None)- if store is None or not question:- return None- if not store_has_semantic_vectors(store):- # Nothing will consume it, so do not pay to compute it.- return None-- key = (id(store), question)- cached = _QUESTION_VECTORS.get(key)- if cached is not None:- _QUESTION_VECTORS.move_to_end(key)- return cached[1]-- timeout_s = embed_timeout_s()- try:- vectors = await asyncio.wait_for(store.embed_texts([question]), timeout=timeout_s)- except TimeoutError:- # The A18 case, and the one worth naming separately: the embedder is- # configured, reachable and healthy, and simply did not answer inside- # the budget. Nothing downstream fails, so without this the answer is- # lexical-only and indistinguishable from one that was not.- _record_leg("embed", "timeout")- _log.warning(- "get_answer could not embed the question within %.1fs; retrieval "- "continues without a question vector. Raise it with %s=<seconds>.",- timeout_s,- _EMBED_TIMEOUT_ENV,- )- return None- except Exception:- _record_leg("embed", "error")- _log.warning(- "get_answer could not embed the question up front; each retrieval "- "stage will embed it again",- exc_info=True,- )- return None- if not vectors:- # Backend without an embedder of its own. Not an error — but it means- # every stage pays for its own round-trip, so it is worth seeing.- _log.debug("Vector store cannot embed directly; per-stage embedding stands")- return None-- vector = [float(v) for v in vectors[0]]- _QUESTION_VECTORS[key] = (store, vector)- while len(_QUESTION_VECTORS) > _QUESTION_VECTOR_CACHE_MAX:- _QUESTION_VECTORS.popitem(last=False)- return vector---async def vector_search(- store: Any, question: str, limit: int, *, vector: list[float] | None-) -> list[Any]:- """Nearest pages to *question*, reusing *vector* when the backend allows it.-- Every backend that can search by raw vector is spared a second embedding of- text it has already embedded; one that cannot returns None from- ``search_by_vector`` and is asked to search the text as before.-- Returns nothing on a keyless store. Guarded here rather than only at the- callers because this is the shared floor under every vector read in the- answer pipeline, and the text fallback on the last line would otherwise- route a keyless store straight back into the embedder this is avoiding.- """- if not store_has_semantic_vectors(store):- return []- if vector is not None:- by_vector = await store.search_by_vector(vector, limit=limit)- if by_vector is not None:- return by_vector- return await store.search(question, limit=limit)---# ----------------------------------------------------------------------------# Stage 1: Hybrid retrieval (FTS + vector → RRF merge)-# ------------------------------------------------------------------------------async def hybrid_retrieve(question: str, ctx: Any) -> list[dict]:- """Run FTS, vector, symbol and filename retrieval in parallel and merge via RRF.-- Returns a list of dicts shaped ``{page_id, title, score, snippet,- page_type, _sources: set[str]}``. ``_sources`` names which retrievers- found the hit — useful for ranking signal ("hit by both modes" is a- stronger ground-truth signal than "hit by one"). Score is the RRF-fused- score; downstream stages may multiply it further.-- Both retrievers are best-effort with timeouts so one slow path can never- block the call. An empty result from one mode just means the other mode- fully drives ranking, which matches the pre-hybrid behaviour.-- The symbol and filename legs also name indexed files that have no page, as- ``page_type: "file"`` hits, the same rows ``search_codebase`` serves.- """- # Reset before the legs run so the record describes this question and not- # a previous one that happened to share the task context.- begin_leg_record()- fts_task = _safe_fts_search(ctx, question)- vec_task = _safe_vector_search(ctx, question)- sym_task = _safe_symbol_search(ctx, question, pageless=True)- name_task = _safe_filename_search(ctx, question)- fts_results, vec_results, sym_results, name_results = await asyncio.gather(- fts_task, vec_task, sym_task, name_task- )-- # RRF merge. Each hit's contribution from a source is 1/(rank + k);- # hits appearing in both sources sum their contributions naturally.- # Per-source rank is preserved alongside the fused score: RRF *compresses*- # scores (rank-0-in-both barely outscores rank-1-in-both), so the summed- # number loses the "both retrievers independently ranked this #1" signal.- # Downstream confidence uses these ranks to recover retriever *agreement*- # as a dominance signal the numeric ratio can't see.- fused: dict[str, dict] = {}- for rank, h in enumerate(fts_results):- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)- entry["_sources"].add("fts")- entry["_fts_rank"] = rank- for rank, h in enumerate(vec_results):- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)- entry["_sources"].add("vector")- entry["_vec_rank"] = rank- for rank, h in leg_ranks(sym_results):- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _SYMBOL_LEG_RRF_K)- entry["_sources"].add("symbol")- entry["_sym_rank"] = rank- entry["_symbol_names"] = h.symbol_names- # Adds files the other legs missed and never reorders a page they found.- # Search fuses the same rows into its own entry shape, crediting a symbol- # page of the file; here every leg is keyed by page id, as above.- for rank, (h, full) in leg_ranks(name_results):- if h.page_id in fused and h.page_type != PAGELESS_FILE:- continue- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))- entry["score"] = entry.get("score", 0.0) + 1.0 / (- rank + (_RRF_K if full else _FILENAME_LEG_RRF_K)- )- entry["_sources"].add("filename")-- # Scale to BM25-range so downstream confidence/dominance gates (tuned- # against the prior single-mode BM25 retrieval) keep behaving sanely.- # Ordering is unchanged — multiplying by a positive constant is a- # no-op for ranking.- for entry in fused.values():- entry["score"] = entry["score"] * _RRF_SCORE_SCALE-- merged = list(fused.values())- merged.sort(key=lambda h: h["score"], reverse=True)- return merged---def stamp_hybrid_rank(hits: list[dict]) -> None:- """Record each hit's 0-based file rank in the fused order, before any rerank.-- This is the order ``search_codebase`` serves; confidence reads it to tell- when the reranked lead left it. Symbol pages share their file's rank. Files- are ranked among pages: a file with no page shares the rank of the page- below it, so it never pushes a page down.- """- files: dict[str, int] = {}- for h in hits:- path = (h.get("target_path") or "").split("::", 1)[0]- if h.get("page_type") == PAGELESS_FILE:- h["_hybrid_rank"] = len(files)- else:- h["_hybrid_rank"] = files.setdefault(path, len(files))---async def _safe_fts_search(ctx: Any, question: str) -> list[Any]:- """FTS search wrapped in timeout + suppression. Returns [] on any failure."""- if ctx.fts is None:- _record_leg("fts", "absent")- return []- try:- results = await asyncio.wait_for(- ctx.fts.search(question, limit=_RETRIEVAL_FETCH_LIMIT), timeout=5.0- )- except TimeoutError:- _record_leg("fts", "timeout")- return []- except Exception:- _record_leg("fts", "error")- return []- _record_leg("fts", "ok")- return results---def _pages_only(results: list[Any], limit: int) -> list[Any]:- """Best-first *results* with decision vectors removed, capped at *limit*.-- Decision records live in the page store under their own page-id namespace so- dedup can match a paraphrase and ``search_codebase`` can surface a decision- directly. Neither is true of answering: a decision row has no ``wiki_pages``- row, so hydration leaves it pathless — and a pathless hit skips the tombstone- check and the scope filter, can only be reordered by noise demotion rather- than dropped, and cannot be cited by the answer it helped write.-- A why-shaped question still gets its decisions. They are injected from a- path-overlap query over ``decision_records``, which does not involve this- store at all.- """- kept = [r for r in results if not str(getattr(r, "page_id", "")).startswith(_DECISION_PREFIX)]- dropped = len(results) - len(kept)- if dropped:- _log.debug(- "get_answer page retrieval dropped %d decision vector(s) from a window of %d",- dropped,- len(results),- )- if len(kept) < limit and dropped:- # The over-fetch margin was not enough. Reported because the visible- # symptom is a short candidate set, which otherwise reads as a corpus- # too small or a query too narrow.- _log.warning(- "get_answer page retrieval kept only %d of a requested %d candidates: %d "- "decision vector(s) filled the window. Decisions may be over-represented "- "in the vector store.",- len(kept),- limit,- dropped,- )- return kept[:limit]---async def _safe_vector_search(ctx: Any, question: str) -> list[Any]:- """Vector search wrapped in timeout + suppression. Returns [] on any failure.-- Also waits for vector-store readiness when the lifespan event is set —- skipping the wait would race a background-loading store on cold start.-- Returns nothing on a keyless index, before the readiness wait and before the- question is embedded: there is no vector worth computing when there is- nothing discriminative to compare it against. See- ``store_has_semantic_vectors``. This leg is the only entry to vector- retrieval here, so guarding it covers every caller.- """- if ctx.vector_store is None:- _record_leg("vector", "absent")- return []- if not store_has_semantic_vectors(ctx.vector_store):- _record_leg("vector", "keyless")- return []- ready = getattr(ctx, "vector_store_ready", None)- if ready is not None:- with contextlib.suppress(TimeoutError):- await asyncio.wait_for(ready.wait(), timeout=30.0)- # Embedded here, before the store is asked anything: this is the first stage- # to need the vector, and every later stage reads the same one back.- vector = await question_vector(ctx, question)- try:- results = await asyncio.wait_for(- vector_search(- ctx.vector_store,- question,- _RETRIEVAL_FETCH_LIMIT + _DECISION_OVERFETCH,- vector=vector,- ),- timeout=vector_search_timeout_s(),- )- except TimeoutError:- _record_leg("vector", "timeout")- _log.warning(- "Vector search exceeded its %gs budget; answering without semantic hits. "- "Raise it with %s=<seconds>.",- vector_search_timeout_s(),- _VECTOR_TIMEOUT_ENV,- )- return []- except Exception:- _record_leg("vector", "error")- return []- _record_leg("vector", "ok")- return _pages_only(results, _RETRIEVAL_FETCH_LIMIT)---class _SymbolLegResult:- """A file page reached through the symbol index, in retriever result shape.-- The RRF merge is keyed on page ids and reads four attributes off whatever- the retrievers hand it, so the symbol leg presents its file pages the same- way FTS and the vector store present theirs.- """-- __slots__ = ("page_id", "page_type", "snippet", "symbol_names", "title")-- def __init__(- self, page_id: str, title: str, snippet: str, page_type: str, symbol_names: list[str]- ) -> None:- self.page_id = page_id- self.title = title- self.snippet = snippet- self.page_type = page_type- self.symbol_names = symbol_names---async def _safe_symbol_search(- ctx: Any, question: str, *, pageless: bool = False-) -> list[_SymbolLegResult]:- """File pages whose symbols the question's words name. [] on any failure.-- Ungated: it runs on every question, not only on ones that happen to carry- an identifier-shaped token. Which indexes get read is not something the- grammar of the sentence should decide. "How does an incremental update- persist symbols" and ``_persist_symbols`` are after the same file, and- before this leg only the second one reached the symbol index.-- Best-effort with a timeout, like the other two legs: a slow or missing- symbol index degrades ``get_answer`` to its previous behaviour rather than- failing the call. ``pageless`` keeps files that have no page, as pageless- rows, for a caller that can serve them.- """- try:- pages = await asyncio.wait_for(- symbol_backed_pages(- ctx,- question,- max_files=_SYMBOL_LEG_MAX_PAGES,- symbol_limit=_SYMBOL_LEG_FETCH_LIMIT,- pageless=pageless,- ),- timeout=5.0,- )- except TimeoutError:- _record_leg("symbol", "timeout")- _log.debug("get_answer symbol leg timed out; page retrieval stands")- return []- except Exception:- _record_leg("symbol", "error")- _log.debug("get_answer symbol leg failed; page retrieval stands", exc_info=True)- return []- _record_leg("symbol", "ok")- return [- _SymbolLegResult(- p["page_id"],- p["title"],- (p.get("summary") or "")[:200],- p["page_type"],- p.get("symbol_names") or [],- )- for p in pages- ]---async def _safe_filename_search(ctx: Any, question: str) -> list[tuple[_SymbolLegResult, bool]]:- """Files whose file name the question spells, each with whether it spells all- of the name. [] on any failure, like the other legs."""- try:- pages = await asyncio.wait_for(- filename_backed_pages(ctx, question, max_files=_SYMBOL_LEG_MAX_PAGES), timeout=5.0- )- except TimeoutError:- _record_leg("filename", "timeout")- return []- except Exception:- _record_leg("filename", "error")- _log.debug("filename leg failed; page retrieval stands", exc_info=True)- return []- _record_leg("filename", "ok")- return [- (- _SymbolLegResult(p["page_id"], p["title"], p["summary"][:200], p["page_type"], []),- p["full_cover"],- )- for p in pages- ]---def leg_ranks(results: list[Any]) -> list[tuple[int, Any]]:- """Each result of a leg with its rank among that leg's pages.-- A file with no page takes the rank of the page below it, so it never costs- a page its place in the leg: the pages fuse exactly as they would without it.- Items may be results or ``(result, extra)`` pairs.- """- ranked: list[tuple[int, Any]] = []- pages = 0- for item in results:- result = item[0] if isinstance(item, tuple) else item- ranked.append((pages, item))- if result.page_type != PAGELESS_FILE:- pages += 1- return ranked---def _hit_dict_from_result(result: Any) -> dict:- """Convert a retriever result object to the pipeline's dict shape."""- return {- "page_id": result.page_id,- "title": getattr(result, "title", ""),- "snippet": getattr(result, "snippet", ""),- "page_type": getattr(result, "page_type", ""),- "score": 0.0,- "_sources": set(),- }---# ----------------------------------------------------------------------------# Noise demotion (decision records + test file pages)-# -----------------------------------------------------------------------------# Retrieval noise that should not occupy get_answer's top-5 on a plain question.-# Mirrors search_codebase's demotion (tool_search._sort_demoting_noise) on the-# answer pipeline's hit shape. Both now read the same test-path rules, so the-# two tools can no longer demote different sets of files.-_TEST_QUERY_RE = re.compile(- r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",- re.IGNORECASE,-)---def demote_noise_hits(hits: list[dict], question: str, *, is_why: bool) -> list[dict]:- """Stable-partition retrieval noise below real pages before the top-5 cap.-- get_answer applies no demotion of its own — decision records (short dense- titles) and test file pages win RRF against the implementation a plain- question is about and take top-5 slots that then feed synthesis. Decision- records demote unless the question is why-shaped (decisions are the answer- then, folded into the prelude); test file pages demote unless the question is- explicitly about tests. Stable: real hits keep their order, and noise keeps- its relative order at the tail (never dropped — an agent may still want it).-- Tests demote, test support does not: "where are the shared fixtures" is a- plain question whose answer is a ``conftest.py``, and demoting it would push- the answer out of the window that feeds synthesis.- """- if not hits:- return hits- test_focused = bool(_TEST_QUERY_RE.search(question))-- def _is_noise(h: dict) -> bool:- pt = h.get("page_type")- return (pt == "decision_record" and not is_why) or (- pt in FILE_ROW_TYPES and not test_focused and is_test_path(h.get("target_path") or "")- )-- real = [h for h in hits if not _is_noise(h)]- noise = [h for h in hits if _is_noise(h)]- return real + noise---# ----------------------------------------------------------------------------# Stage 2: Hydrate hits with Page metadata (target_path, summary)-# ------------------------------------------------------------------------------async def hydrate_hits(hits: list[dict], ctx: Any, *, scope: str | None = None) -> list[dict]:- """Attach target_path, summary, and page_type from the Page table.-- Mutates each hit in place. Applies the ``scope`` filter (path prefix) at- this stage rather than during retrieval — retrievers don't know about- paths, and post-filtering keeps the merge logic source-agnostic.- """- if not hits:- return hits- page_ids = [h["page_id"] for h in hits]- async with get_session(ctx.session_factory) as session:- res = await session.execute(- select(- Page.id,- Page.target_path,- Page.summary,- Page.page_type,- Page.freshness_status,- ).where(Page.id.in_(page_ids))- )- meta_by_id = {- row[0]: {- "target_path": row[1] or "",- "summary": row[2] or "",- "page_type": row[3] or "",- "freshness": row[4] or "",- }- for row in res.all()- }-- out: list[dict] = []- pageless = 0- for h in hits:- meta = meta_by_id.get(h["page_id"])- if meta is None and h.get("page_type") == PAGELESS_FILE:- # An indexed file with no page: its path is in its id, and it has- # no summary to give, so it serves its matched code instead.- meta = {"target_path": pageless_path(h["page_id"]), "page_type": PAGELESS_FILE}- if meta is None:- # A retrieved id with no page behind it: a decision vector, or a- # vector left over from a page the stores have since disagreed- # about. Either way there is nothing to cite and nothing to read,- # and keeping it costs a served slot — while every later gate that- # would have caught it (tombstone, scope) is keyed on a path it- # does not have.- pageless += 1- continue- # Tombstoned pages document deleted/renamed files — serving them as- # answer material would cite code that no longer exists.- if meta.get("freshness") == "tombstone":- continue- target_path = meta.get("target_path", "")- if scope and target_path and not target_path.startswith(scope):- continue- h["target_path"] = target_path- h["summary"] = meta.get("summary", "")- # Prefer the Page table's page_type when present — it's the source- # of truth; retrievers sometimes carry stale or empty types.- h["page_type"] = meta.get("page_type") or h.get("page_type", "")- out.append(h)- if pageless:- _log.warning(- "get_answer dropped %d of %d retrieved hit(s) with no page row behind them; "- "an id in a retriever that the page table does not know is either a "- "non-page vector or a three-store disagreement (repowise doctor --repair)",- pageless,- len(hits),- )- return out---# ----------------------------------------------------------------------------# Stage 3: PageRank bias-# ------------------------------------------------------------------------------async def apply_pagerank_bias(hits: list[dict], ctx: Any) -> None:- """Multiply each hit's score by a damped PageRank factor (in place).-- Looks up the GraphNode row for each hit's target_path and pulls its- PageRank. We normalise within the candidate set rather than against the- whole graph: a candidate with the highest PageRank among its peers gets- the full bias, the lowest gets none. This avoids the failure mode where- an absolute scale would over-reward famous-but-irrelevant files.- """- if not hits:- return- paths = [h.get("target_path") for h in hits if h.get("target_path")]- if not paths:- return- async with get_session(ctx.session_factory) as session:- # Look up GraphNodes by node_id — file nodes are keyed by their path.- res = await session.execute(- select(GraphNode.node_id, GraphNode.pagerank).where(- GraphNode.node_id.in_(paths),- GraphNode.node_type == "file",- )- )- pr_by_path = {row[0]: float(row[1] or 0.0) for row in res.all()}-- if not pr_by_path:- return- # Normalised over the pages, so a file with no page cannot change their bias.- paged = {h.get("target_path") for h in hits if h.get("page_type") != PAGELESS_FILE}- max_pr = max((pr for path, pr in pr_by_path.items() if path in paged), default=0.0)- max_pr = max_pr or max(pr_by_path.values(), default=0.0)- if max_pr <= 0:- return-- for h in hits:- pr = pr_by_path.get(h.get("target_path"), 0.0)- # Normalised in [0, 1] then scaled to a multiplicative bias in- # [1.0, 1 + _PAGERANK_BIAS_MAX].- bias = 1.0 + _PAGERANK_BIAS_MAX * min(pr / max_pr, 1.0)- h["_pagerank"] = pr- h["_pagerank_bias"] = round(bias, 3)- h["score"] = h.get("score", 0.0) * bias-- hits.sort(key=lambda h: h["score"], reverse=True)---# ----------------------------------------------------------------------------# Stage 4: Graph expansion (1-hop neighbors of top hits)-# ------------------------------------------------------------------------------def _projected_pair(- src: str | None, tgt: str | None, edge_type: str | None, confidence: float | None-) -> tuple[str | None, str | None]:- """An edge's two endpoints as file paths, or ``(None, None)`` to drop it.-- Only the symbol layer gets the guards; the file layer is already- file-to-file and keeps today's behaviour. The asymmetry is deliberate — the- guards refuse cross-extension pairs, and a ``co_changes`` edge between a- module and its docs is a "read this too" expansion has always served.- """- src = src or ""- tgt = tgt or ""- if not is_symbol_node(src) and not is_symbol_node(tgt):- return (src or None, tgt or None)- src_file, tgt_file = node_to_file(src), node_to_file(tgt)- if not keep_projected_edge(src_file, tgt_file, edge_type, confidence):- return (None, None)- return (src_file, tgt_file)---async def _projected_edges(session: Any, repo_id: str) -> list[tuple[str, str]]:- """Every non-containment edge for *repo_id*, as a pair of file paths.-- One scan, projected in Python. A prefix match on the symbol layer cannot use- the node-id index (SQLite disables the LIKE optimisation under an ESCAPE- clause), so per-candidate queries were a scan each; this is one.-- Containment is excluded because both its endpoints describe the same file,- so a projected containment edge is always a self-loop.- """- res = await session.execute(- select(- GraphEdge.source_node_id,- GraphEdge.target_node_id,- GraphEdge.edge_type,- GraphEdge.confidence,- ).where(- GraphEdge.repository_id == repo_id,- GraphEdge.edge_type.notin_(CONTAINMENT_EDGE_TYPES),- )- )- pairs: list[tuple[str, str]] = []- for src, tgt, etype, conf in res.all():- src_file, tgt_file = _projected_pair(src, tgt, etype, conf)- if src_file and tgt_file:- pairs.append((src_file, tgt_file))- return pairs---def _hub_degree_cutoff(degree: dict[str, int]) -> int:- """Degree at which a candidate counts as a hub, for this candidate set.-- ``max(floor, p99)``. The floor keeps a small or sparsely-linked repo from- excluding ordinary files, and the percentile keeps a densely-linked one- from excluding nothing.- """- if not degree:- return 1 << 30- values = sorted(degree.values())- idx = min(len(values) - 1, int(len(values) * _HUB_DEGREE_PERCENTILE))- return max(_HUB_DEGREE_FLOOR, values[idx])---async def expand_via_graph(hits: list[dict], ctx: Any, repo_id: str) -> list[dict]:- """Add up to ``_GRAPH_EXPAND_MAX_NEW`` graph-neighbor files to ``hits``.-- Rescues near-misses where the top retrieved file is in the right- neighborhood but isn't the actual answer (a consumer instead of the- orchestrator, a wrapper instead of the implementation). Expansion walks- one hop through GraphEdges in both directions from the top-N candidates,- then folds in any neighbor that:- * has a wiki page (otherwise the LLM has nothing to read)- * is not already in the candidate set-- Expanded hits carry an ``_expanded_from`` marker and a damped score so- the gate / confidence calibration knows they're indirect.- """- if not hits:- return hits- # Seeded from pages only, so a file with no page cannot change what pages add.- seeds = [h for h in hits if h.get("page_type") != PAGELESS_FILE][:_GRAPH_EXPAND_TOP_N]- seed_paths = [h.get("target_path") for h in seeds if h.get("target_path")]- if not seed_paths:- return hits- existing = {h.get("target_path") for h in hits}-- async with get_session(ctx.session_factory) as session:- # Not import edges only, despite the naming this code used to carry:- # ``co_changes`` neighbours stay in on purpose, because "this file moves- # with yours" is a genuine read-this-too signal, and expansion surfaces- # what it adds neutrally as ``[graph-expanded]`` rather than as an import- # claim.- #- # The scan reaches the symbol layer as well as the file layer: ``calls``- # and its siblings join ``path::Name`` nodes, so an equality test against- # a seed path matched none of them and the call graph was invisible here.- seed_set = set(seed_paths)- pairs = await per_index(- session, repo_id, "projected_edges", lambda: _projected_edges(session, repo_id)- )-- neighbors: set[str] = set()- degree: dict[str, int] = {}- for src_file, tgt_file in pairs:- degree[src_file] = degree.get(src_file, 0) + 1- degree[tgt_file] = degree.get(tgt_file, 0) + 1- for mine, theirs in ((src_file, tgt_file), (tgt_file, src_file)):- if mine in seed_set and theirs not in existing:- neighbors.add(theirs)-- if not neighbors:- return hits-- # Only fold in neighbors that have a wiki page — otherwise the LLM- # context block can't carry a useful excerpt for them.- page_res = await session.execute(- select(Page.target_path, Page.summary, Page.page_type).where(- Page.target_path.in_(sorted(neighbors)),- Page.page_type == "file_page",- )- )- page_rows = list(page_res.all())-- # Also load PageRank for the neighbors so we can rank them.- pr_res = await session.execute(- select(GraphNode.node_id, GraphNode.pagerank).where(- GraphNode.node_id.in_(sorted(neighbors)),- GraphNode.node_type == "file",- )- )- pr_by_path = {row[0]: float(row[1] or 0.0) for row in pr_res.all()}-- if not page_rows:- return hits-- # Drop hubs before ranking. Ranking by PageRank alone actively prefers- # them, which is the wrong instinct here: expansion is trying to name the- # specific file the question is about, and the most-imported file in the- # repo is the least specific candidate available.- # Scoped to the candidate set, not the repo: the cutoff answers "is this- # neighbour a hub among the ones on offer".- cutoff = _hub_degree_cutoff({row[0]: degree.get(row[0], 0) for row in page_rows})- non_hub = [row for row in page_rows if degree.get(row[0], 0) <= cutoff]- if non_hub:- page_rows = non_hub-- # Damp parent score by _GRAPH_EXPAND_DAMPING for child candidates; pick- # the strongest parent each child connects to (taking the max parent- # score is conservative — favors well-connected neighbors).- strongest_parent = max(- seeds, key=lambda hit: hit.get("score", 0.0), default={}- )- parent_score = strongest_parent.get("score", 0.0)- confidence_factor = strongest_parent.get("_confidence_score_factor")- candidates: list[dict] = []- for path, summary, page_type in page_rows:- candidate = {- "page_id": f"file_page:{path}",- "target_path": path,- "title": f"File: {path}",- "summary": summary or "",- "snippet": (summary or "")[:200],- "page_type": page_type or "file_page",- "score": parent_score * _GRAPH_EXPAND_DAMPING,- "_sources": {"graph_expand"},- "_expanded_from": "graph",- "_pagerank": pr_by_path.get(path, 0.0),- }- if confidence_factor is not None:- candidate["_confidence_score_factor"] = confidence_factor- candidates.append(candidate)-- # Rank candidates by PageRank within the expansion set so we pick the- # most central neighbor first when we have multiple plausible ones.- # Path breaks ties, so which neighbour is picked cannot move with the- # database's row order between runs on identical data.- candidates.sort(key=lambda c: (-c.get("_pagerank", 0.0), c["target_path"]))- additions = candidates[:_GRAPH_EXPAND_MAX_NEW]- if not additions:- return hits-- combined = hits + additions- combined.sort(key=lambda h: h["score"], reverse=True)- return combined---# ----------------------------------------------------------------------------# Stage: parent-concept surfacing (subsystem-shaped questions)-# -----------------------------------------------------------------------------# A subsystem-shaped question asks about a part of the system as a whole:-# "overview of X", "main parts of X", "what subsystem does Y belong to",-# "where would I add a Z". For these the best answer is the concept page that-# documents the whole subsystem, not one of its member files or a more specific-# child concept page. Retrieval ranks the specific children above the parent (a-# child's embedding matches the query's noun more tightly than the broader-# parent), so the parent concept/rollup page never surfaces even though it-# exists. This gate is purely on the query's natural-language shape — no-# repo-specific vocabulary — so it generalises across codebases.-# Kept deliberately high-precision: bare fragments like "parts of", "belongs to",-# "structure of", or "which module" appear constantly inside ordinary how/where-# questions ("what parts of the code touch this", "which module imports config"),-# so each phrase is anchored to an unambiguous subsystem-overview intent. Better-# to miss an oddly-worded subsystem question (it just keeps today's ranking) than-# to reorder an implementation question's file hits.-_SUBSYSTEM_QUERY_RE = re.compile(- r"\boverview of\b|\bgive me an overview\b|\bhigh[- ]level\b|"- r"\bmain parts of\b|\bwhat (?:are|is) the (?:main )?(?:parts|pieces|components) of\b|"- r"\barchitecture of\b|\bwalk me through\b|"- r"\bhow (?:is|are|does) .+ (?:organi[sz]ed|structured|laid out|put together)\b|"- r"\bwhat subsystem\b|\bwhich subsystem\b|"- r"\bwhere would i (?:add|put)\b|\bwhere do i (?:add|put)\b",- re.IGNORECASE,-)--# How many of the strongest real (non-noise) hits to cluster when looking for-# their shared subsystem. Decision records and pages without a path are skipped-# so a query whose top slots are crowded by decision noise still clusters on its-# real member files.-_PARENT_EXPAND_TOP_N = 8-# A directory is a tight structural cluster when it is the immediate parent of at-# least this many surfaced hits. A lone surfaced file is not enough structural-# signal on its own; that case is left to the semantic concept lookup.-_PARENT_MIN_SHARE = 2-# The injected parent leads the surface (it is the answer) but only just: a-# small multiplier over the current top score keeps it first without-# manufacturing a dominant retrieval that would inflate confidence to "high".-_PARENT_EXPAND_BOOST = 1.02-# How deep to look in a concept-restricted vector search for the subsystem page-# a query is semantically about. The window is wide because concept pages are a-# small minority of the corpus, so the right one can sit below many file/symbol-# hits before this filter drops those away.-_CONCEPT_FETCH_LIMIT = 60---def is_subsystem_query(question: str) -> bool:- """True when the question asks about a subsystem/module as a whole, so the- concept page for that subsystem should lead rather than its member files."""- return bool(_SUBSYSTEM_QUERY_RE.search(question or ""))---def _common_ancestor(paths: set[str]) -> str:- """Longest shared directory prefix of the given paths, segment-wise."""- split = [p.split("/") for p in paths]- common: list[str] = []- for segs in zip(*split, strict=False):- if len(set(segs)) == 1:- common.append(segs[0])- else:- break- return "/".join(common)---async def _semantic_concept_paths(question: str, ctx: Any) -> list[str]:- """Target paths of the concept/layer pages a concept-restricted vector search- ranks highest for the question, best-first.-- This is the "guaranteed concept-page candidate": when a subsystem query's- file hits land in the wrong neighborhood (a UI consumer of the subsystem- rather than the subsystem itself), the subsystem's own concept page still- matches the query semantically and surfaces here even though it never- entered the file-dominated main fetch. Best-effort: returns [] on any error- so the caller degrades to the structural path.- """- vs = getattr(ctx, "vector_store", None)- if vs is None:- return []- try:- results = await asyncio.wait_for(- vector_search(- vs, question, _CONCEPT_FETCH_LIMIT, vector=await question_vector(ctx, question)- ),- timeout=vector_search_timeout_s(),- )- except Exception:- return []- out: list[str] = []- for r in results:- if getattr(r, "page_type", "") in ("module_page", "layer_page"):- tp = getattr(r, "target_path", "") or ""- if not tp:- pid = getattr(r, "page_id", "") or ""- tp = pid.split(":", 1)[1] if ":" in pid else pid- if tp and tp not in out:- out.append(tp)- return out---async def expand_via_parent_page(hits: list[dict], question: str, ctx: Any) -> list[dict]:- """Lead a subsystem-shaped question with the concept page for its subsystem.-- Two complementary signals pick that page, neither tuned to any repository:-- * Structural — when the surfaced file hits cluster under an ancestor- directory that has a concept/rollup page, that ancestor IS the subsystem.- Ancestors are found by walking each hit's target_path up the tree (the- same relationship the generator uses to mint rollups). The tightest- (deepest) ancestor covering >=2 hits wins, never a catch-all near the root.- * Semantic — when the file hits land in the wrong neighborhood (a consumer- of the subsystem, not the subsystem), the subsystem's own concept page- still matches the query and is recovered by a concept-restricted vector- search. Used only when no structural cluster is found, so a strong file- cluster is never overridden by a semantic guess.-- The chosen page is promoted in place if already retrieved (its children- out-embed it, so the cap drops it) or injected as the leading hit otherwise.- A no-op on every non-subsystem question, so file/implementation queries keep- today's ranking untouched.- """- if not hits or not is_subsystem_query(question):- return hits- # Cluster on the strongest real hits: skip decision records and any hit- # without a path so decision noise crowding the top slots can't starve the- # clustering of the member files that reveal the subsystem.- top = [h for h in hits if h.get("target_path") and h.get("page_type") != "decision_record"][- :_PARENT_EXPAND_TOP_N- ]- if not top:- return hits-- # For each surfaced hit, count its immediate parent directory and credit- # every strict ancestor with covering it. A dir that is the immediate parent- # of two or more surfaced hits is a TIGHT cluster: the query's own member- # files sit directly in it. Coverage by a distant ancestor (a broad root that- # merely contains scattered hits) is deliberately not enough — that is what- # separates a real subsystem from the repository root.- imm_count: dict[str, int] = {}- covers: dict[str, set[str]] = {}- for h in top:- tp = h["target_path"].rstrip("/")- parent = tp.rsplit("/", 1)[0] if "/" in tp else ""- if parent:- imm_count[parent] = imm_count.get(parent, 0) + 1- anc = parent- while anc:- covers.setdefault(anc, set()).add(tp)- anc = anc.rsplit("/", 1)[0] if "/" in anc else ""-- tight_clusters = {d for d, c in imm_count.items() if c >= _PARENT_MIN_SHARE}- semantic_paths = await _semantic_concept_paths(question, ctx)- candidates = set(covers) | set(imm_count) | set(semantic_paths)- if not candidates:- return hits-- async with get_session(ctx.session_factory) as session:- rows = (- await session.execute(- select(Page.target_path, Page.title, Page.summary, Page.page_type).where(- Page.target_path.in_(candidates),- Page.page_type.in_(("module_page", "layer_page")),- )- )- ).all()- if not rows:- return hits- by_path = {r[0]: r for r in rows}-- # A tight structural cluster is the query's own files agreeing on a subsystem,- # so it outranks the semantic guess. When several sibling dirs each cluster- # (a subsystem split into subdirectories), roll up to their common ancestor- # page so the answer is the subsystem, not one arbitrary half of it. With no- # tight cluster the file hits landed in the wrong neighborhood, so the concept- # page the query is semantically about wins instead.- tight_pages = tight_clusters & by_path.keys()- winner = None- if tight_pages:- rollup = _common_ancestor(tight_pages)- if len(tight_pages) > 1 and rollup in by_path and rollup in covers:- winner = by_path[rollup]- else:- best_tp = max(tight_pages, key=lambda d: (imm_count[d], d.count("/")))- winner = by_path[best_tp]- else:- # No tight cluster: the semantically-closest concept page (best-first),- # then any weak cover ancestor as a last resort.- winner = next((by_path[tp] for tp in semantic_paths if tp in by_path), None)- if winner is None:- cov_pages = [a for a in covers if a in by_path]- if cov_pages:- winner = by_path[max(cov_pages, key=lambda a: (a.count("/"), len(covers[a])))]- if winner is None:- return hits-- best_tp, best_title, best_summary, best_pt = winner- top_score = max((h.get("score", 0.0) for h in hits), default=0.0) or 1.0- lead_score = top_score * _PARENT_EXPAND_BOOST-- # Already in the candidate set but ranked below its own children (they embed- # tighter to the query noun than the broader parent), so the top-5 cap drops- # it. Promote it to lead in place rather than adding a duplicate. The bump is- # small (just past the top hit) so it never manufactures a dominant retrieval- # that would read "high".- for h in hits:- if h.get("target_path") == best_tp:- if h.get("score", 0.0) < lead_score:- h["score"] = lead_score- src = h.get("_sources")- if isinstance(src, set):- src.add("parent_promote")- hits.sort(key=lambda x: x.get("score", 0.0), reverse=True)- return hits-- # Not retrieved at all: inject it as the leading hit.- parent_hit = {- "page_id": f"{best_pt}:{best_tp}",- "target_path": best_tp,- "title": best_title or f"Overview: {best_tp}",- "summary": best_summary or "",- "snippet": (best_summary or "")[:200],- "page_type": best_pt or "module_page",- "score": lead_score,- "_sources": {"parent_expand"},- "_expanded_from": "parent",- }- combined = [parent_hit, *hits]- combined.sort(key=lambda h: h["score"], reverse=True)- return combineddiff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.pynew file mode 100644index 000000000..db1b1bf5f--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.py@@ -0,0 +1,420 @@+"""Retrieval pipeline for ``get_answer``: question in, ranked wiki hits out.++The LLM step, cache and response shape live in ``tool_answer``. Stages, each a+function over hit dicts:++1. ``hybrid_retrieve``: FTS, vector, symbol and filename legs in parallel,+ fused by Reciprocal Rank Fusion. The page legs only see what a generated+ page spells out; the symbol and filename legs recover private helpers and+ local names.+2. ``hydrate_hits`` (``hydrate``): attach path, summary and page type.+3. ``apply_pagerank_bias`` (``hydrate``): central files win near-ties.+4. ``expand_via_graph`` (``graph_expand``): add 1-hop neighbours of the top hits.+5. ``expand_via_parent_page`` (``parent_page``): a subsystem question leads with+ its concept page.++``noise`` demotes decision and test pages; ``vectors`` and ``leg_record`` are+shared by the legs. The legs and their merge live here, so a leg replaced on+this module is the one ``hybrid_retrieve`` calls.+"""++from __future__ import annotations++import asyncio+import contextlib+import logging+from typing import Any++from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX+from repowise.core.providers.embedding import store_has_semantic_vectors+from repowise.server.mcp_server._answer_pipeline.graph_expand import (+ _GRAPH_EXPAND_DAMPING,+ _GRAPH_EXPAND_MAX_NEW,+ _GRAPH_EXPAND_TOP_N,+ _HUB_DEGREE_FLOOR,+ _HUB_DEGREE_PERCENTILE,+ _hub_degree_cutoff,+ _projected_edges,+ _projected_pair,+ expand_via_graph,+)+from repowise.server.mcp_server._answer_pipeline.hydrate import (+ _PAGERANK_BIAS_MAX,+ apply_pagerank_bias,+ hydrate_hits,+)+from repowise.server.mcp_server._answer_pipeline.leg_record import (+ _LEG_RECORD,+ _record_leg,+ begin_leg_record,+ degraded_legs,+ retrieval_legs,+)+from repowise.server.mcp_server._answer_pipeline.noise import _TEST_QUERY_RE, demote_noise_hits+from repowise.server.mcp_server._answer_pipeline.parent_page import (+ _CONCEPT_FETCH_LIMIT,+ _PARENT_EXPAND_BOOST,+ _PARENT_EXPAND_TOP_N,+ _PARENT_MIN_SHARE,+ _SUBSYSTEM_QUERY_RE,+ _common_ancestor,+ _semantic_concept_paths,+ expand_via_parent_page,+ is_subsystem_query,+)+from repowise.server.mcp_server._answer_pipeline.vectors import (+ _QUESTION_VECTOR_CACHE_MAX,+ _QUESTION_VECTORS,+ question_vector,+ vector_search,+)+from repowise.server.mcp_server._helpers import _VECTOR_TIMEOUT_ENV, vector_search_timeout_s+from repowise.server.mcp_server._page_paths import PAGELESS_FILE+from repowise.server.mcp_server._prose_symbols import filename_backed_pages, symbol_backed_pages++_log = logging.getLogger("repowise.mcp.answer")++# Candidates each leg fetches before fusion.+_RETRIEVAL_FETCH_LIMIT = 15++# Decision vectors share the page store under this page-id prefix.+_DECISION_PREFIX = DECISION_VECTOR_PREFIX++# Extra rows the vector leg fetches so dropping decision rows still leaves a+# full window; decision rows rank well on question-shaped text.+_DECISION_OVERFETCH = 15++# Standard RRF k: adjacent ranks score close, a one-leg rank-10 hit still counts.+_RRF_K = 60++# Scales fused scores into the BM25 range the downstream confidence gates were+# tuned on (top RRF 1/60 + 1/60 ~ 0.033 -> ~6). Never changes the order.+_RRF_SCORE_SCALE = 180.0++# Symbol leg size: fewer than the page legs, it adds what they cannot see.+_SYMBOL_LEG_FETCH_LIMIT = 20+_SYMBOL_LEG_MAX_PAGES = 8++# A larger k for name-match legs: they can lift or add a file, never outvote+# two page legs that agree.+_SYMBOL_LEG_RRF_K = 180++# A name the question spells in full (``full_cover``) fuses at the page legs' k.+_FILENAME_LEG_RRF_K = _SYMBOL_LEG_RRF_K+++async def hybrid_retrieve(question: str, ctx: Any) -> list[dict]:+ """Run the four legs in parallel and fuse them by RRF, best first.++ Hits are ``{page_id, title, score, snippet, page_type, _sources}`` plus each+ leg's rank (``_fts_rank``, ``_vec_rank``, ``_sym_rank``): RRF compresses+ scores, and confidence reads leg agreement from the ranks. A leg that fails+ or times out contributes nothing. Pageless files come back as+ ``page_type: "file"`` hits.+ """+ # Reset so the record describes this question, not one sharing the context.+ begin_leg_record()+ fts_task = _safe_fts_search(ctx, question)+ vec_task = _safe_vector_search(ctx, question)+ sym_task = _safe_symbol_search(ctx, question, pageless=True)+ name_task = _safe_filename_search(ctx, question)+ fts_results, vec_results, sym_results, name_results = await asyncio.gather(+ fts_task, vec_task, sym_task, name_task+ )++ fused: dict[str, dict] = {}+ for rank, h in enumerate(fts_results):+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)+ entry["_sources"].add("fts")+ entry["_fts_rank"] = rank+ for rank, h in enumerate(vec_results):+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)+ entry["_sources"].add("vector")+ entry["_vec_rank"] = rank+ for rank, h in leg_ranks(sym_results):+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _SYMBOL_LEG_RRF_K)+ entry["_sources"].add("symbol")+ entry["_sym_rank"] = rank+ entry["_symbol_names"] = h.symbol_names+ # Adds files the other legs missed; never reorders a page they found.+ for rank, (h, full) in leg_ranks(name_results):+ if h.page_id in fused and h.page_type != PAGELESS_FILE:+ continue+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))+ entry["score"] = entry.get("score", 0.0) + 1.0 / (+ rank + (_RRF_K if full else _FILENAME_LEG_RRF_K)+ )+ entry["_sources"].add("filename")++ for entry in fused.values():+ entry["score"] = entry["score"] * _RRF_SCORE_SCALE++ merged = list(fused.values())+ merged.sort(key=lambda h: h["score"], reverse=True)+ return merged+++def stamp_hybrid_rank(hits: list[dict]) -> None:+ """Record each hit's 0-based file rank in the fused order, before any rerank.++ Symbol pages share their file's rank; a pageless file shares the rank of the+ page below it, so it never pushes a page down.+ """+ files: dict[str, int] = {}+ for h in hits:+ path = (h.get("target_path") or "").split("::", 1)[0]+ if h.get("page_type") == PAGELESS_FILE:+ h["_hybrid_rank"] = len(files)+ else:+ h["_hybrid_rank"] = files.setdefault(path, len(files))+++async def _safe_fts_search(ctx: Any, question: str) -> list[Any]:+ """FTS leg; [] on timeout or failure."""+ if ctx.fts is None:+ _record_leg("fts", "absent")+ return []+ try:+ results = await asyncio.wait_for(+ ctx.fts.search(question, limit=_RETRIEVAL_FETCH_LIMIT), timeout=5.0+ )+ except TimeoutError:+ _record_leg("fts", "timeout")+ return []+ except Exception:+ _record_leg("fts", "error")+ return []+ _record_leg("fts", "ok")+ return results+++def _pages_only(results: list[Any], limit: int) -> list[Any]:+ """Best-first *results* without decision vectors, capped at *limit*.++ A decision row has no page, so it could not be cited, scoped or tombstoned.+ Why-shaped questions get decisions from ``decision_records`` instead.+ """+ kept = [r for r in results if not str(getattr(r, "page_id", "")).startswith(_DECISION_PREFIX)]+ dropped = len(results) - len(kept)+ if dropped:+ _log.debug(+ "get_answer page retrieval dropped %d decision vector(s) from a window of %d",+ dropped,+ len(results),+ )+ if len(kept) < limit and dropped:+ # A short window would otherwise read as thin retrieval.+ _log.warning(+ "get_answer page retrieval kept only %d of a requested %d candidates: %d "+ "decision vector(s) filled the window. Decisions may be over-represented "+ "in the vector store.",+ len(kept),+ limit,+ dropped,+ )+ return kept[:limit]+++async def _safe_vector_search(ctx: Any, question: str) -> list[Any]:+ """Vector leg; [] on a keyless store, timeout or failure.++ Waits for a store still loading at startup, and checks for a keyless store+ before embedding anything.+ """+ if ctx.vector_store is None:+ _record_leg("vector", "absent")+ return []+ if not store_has_semantic_vectors(ctx.vector_store):+ _record_leg("vector", "keyless")+ return []+ ready = getattr(ctx, "vector_store_ready", None)+ if ready is not None:+ with contextlib.suppress(TimeoutError):+ await asyncio.wait_for(ready.wait(), timeout=30.0)+ vector = await question_vector(ctx, question)+ try:+ results = await asyncio.wait_for(+ vector_search(+ ctx.vector_store,+ question,+ _RETRIEVAL_FETCH_LIMIT + _DECISION_OVERFETCH,+ vector=vector,+ ),+ timeout=vector_search_timeout_s(),+ )+ except TimeoutError:+ _record_leg("vector", "timeout")+ _log.warning(+ "Vector search exceeded its %gs budget; answering without semantic hits. "+ "Raise it with %s=<seconds>.",+ vector_search_timeout_s(),+ _VECTOR_TIMEOUT_ENV,+ )+ return []+ except Exception:+ _record_leg("vector", "error")+ return []+ _record_leg("vector", "ok")+ return _pages_only(results, _RETRIEVAL_FETCH_LIMIT)+++class _SymbolLegResult:+ """A file page from a name leg, in the shape FTS and vector results have."""++ __slots__ = ("page_id", "page_type", "snippet", "symbol_names", "title")++ def __init__(+ self, page_id: str, title: str, snippet: str, page_type: str, symbol_names: list[str]+ ) -> None:+ self.page_id = page_id+ self.title = title+ self.snippet = snippet+ self.page_type = page_type+ self.symbol_names = symbol_names+++async def _safe_symbol_search(+ ctx: Any, question: str, *, pageless: bool = False+) -> list[_SymbolLegResult]:+ """File pages whose symbols the question's words name; [] on any failure.++ Runs on every question, not only identifier-shaped ones. ``pageless`` keeps+ files with no page for a caller that can serve them.+ """+ try:+ pages = await asyncio.wait_for(+ symbol_backed_pages(+ ctx,+ question,+ max_files=_SYMBOL_LEG_MAX_PAGES,+ symbol_limit=_SYMBOL_LEG_FETCH_LIMIT,+ pageless=pageless,+ ),+ timeout=5.0,+ )+ except TimeoutError:+ _record_leg("symbol", "timeout")+ _log.debug("get_answer symbol leg timed out; page retrieval stands")+ return []+ except Exception:+ _record_leg("symbol", "error")+ _log.debug("get_answer symbol leg failed; page retrieval stands", exc_info=True)+ return []+ _record_leg("symbol", "ok")+ return [+ _SymbolLegResult(+ p["page_id"],+ p["title"],+ (p.get("summary") or "")[:200],+ p["page_type"],+ p.get("symbol_names") or [],+ )+ for p in pages+ ]+++async def _safe_filename_search(ctx: Any, question: str) -> list[tuple[_SymbolLegResult, bool]]:+ """Files the question names, each with whether it spells the whole name; [] on failure."""+ try:+ pages = await asyncio.wait_for(+ filename_backed_pages(ctx, question, max_files=_SYMBOL_LEG_MAX_PAGES), timeout=5.0+ )+ except TimeoutError:+ _record_leg("filename", "timeout")+ return []+ except Exception:+ _record_leg("filename", "error")+ _log.debug("filename leg failed; page retrieval stands", exc_info=True)+ return []+ _record_leg("filename", "ok")+ return [+ (+ _SymbolLegResult(p["page_id"], p["title"], p["summary"][:200], p["page_type"], []),+ p["full_cover"],+ )+ for p in pages+ ]+++def leg_ranks(results: list[Any]) -> list[tuple[int, Any]]:+ """Each leg result with its rank among the leg's pages.++ A pageless file takes the rank of the page below it, so pages fuse as they+ would without it. Items may be results or ``(result, extra)`` pairs.+ """+ ranked: list[tuple[int, Any]] = []+ pages = 0+ for item in results:+ result = item[0] if isinstance(item, tuple) else item+ ranked.append((pages, item))+ if result.page_type != PAGELESS_FILE:+ pages += 1+ return ranked+++def _hit_dict_from_result(result: Any) -> dict:+ """Convert a retriever result object to the pipeline's dict shape."""+ return {+ "page_id": result.page_id,+ "title": getattr(result, "title", ""),+ "snippet": getattr(result, "snippet", ""),+ "page_type": getattr(result, "page_type", ""),+ "score": 0.0,+ "_sources": set(),+ }+++__all__ = [+ "_CONCEPT_FETCH_LIMIT",+ "_DECISION_OVERFETCH",+ "_DECISION_PREFIX",+ "_FILENAME_LEG_RRF_K",+ "_GRAPH_EXPAND_DAMPING",+ "_GRAPH_EXPAND_MAX_NEW",+ "_GRAPH_EXPAND_TOP_N",+ "_HUB_DEGREE_FLOOR",+ "_HUB_DEGREE_PERCENTILE",+ "_LEG_RECORD",+ "_PAGERANK_BIAS_MAX",+ "_PARENT_EXPAND_BOOST",+ "_PARENT_EXPAND_TOP_N",+ "_PARENT_MIN_SHARE",+ "_QUESTION_VECTORS",+ "_QUESTION_VECTOR_CACHE_MAX",+ "_RETRIEVAL_FETCH_LIMIT",+ "_RRF_K",+ "_RRF_SCORE_SCALE",+ "_SUBSYSTEM_QUERY_RE",+ "_SYMBOL_LEG_FETCH_LIMIT",+ "_SYMBOL_LEG_MAX_PAGES",+ "_SYMBOL_LEG_RRF_K",+ "_TEST_QUERY_RE",+ "_common_ancestor",+ "_hub_degree_cutoff",+ "_projected_edges",+ "_projected_pair",+ "_record_leg",+ "_safe_filename_search",+ "_safe_fts_search",+ "_safe_symbol_search",+ "_safe_vector_search",+ "_semantic_concept_paths",+ "apply_pagerank_bias",+ "begin_leg_record",+ "degraded_legs",+ "demote_noise_hits",+ "expand_via_graph",+ "expand_via_parent_page",+ "filename_backed_pages",+ "hybrid_retrieve",+ "hydrate_hits",+ "is_subsystem_query",+ "leg_ranks",+ "question_vector",+ "retrieval_legs",+ "stamp_hybrid_rank",+ "vector_search",+]diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.pynew file mode 100644index 000000000..cb843bf9e--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.py@@ -0,0 +1,199 @@+"""Graph expansion: 1-hop neighbours of the top hits join the candidate set."""++from __future__ import annotations++from typing import Any++from sqlalchemy import select++from repowise.core.ingestion.models import CONTAINMENT_EDGE_TYPES+from repowise.core.persistence.database import get_session+from repowise.core.persistence.models import GraphEdge, GraphNode, Page+from repowise.server.mcp_server._graph_files import (+ is_symbol_node,+ keep_projected_edge,+ node_to_file,+ per_index,+)+from repowise.server.mcp_server._page_paths import PAGELESS_FILE++# Seeds to expand from, and the most files expansion may add.+_GRAPH_EXPAND_TOP_N = 2+_GRAPH_EXPAND_MAX_NEW = 3++# A neighbour above max(floor, p99) degree is a hub: near everything, so it+# says nothing about this question, yet PageRank would rank it first.+_HUB_DEGREE_FLOOR = 50+_HUB_DEGREE_PERCENTILE = 0.99++# Expanded hits score below their parent but can pass the tail of the top 5.+_GRAPH_EXPAND_DAMPING = 0.7+++def _projected_pair(+ src: str | None, tgt: str | None, edge_type: str | None, confidence: float | None+) -> tuple[str | None, str | None]:+ """An edge's two endpoints as file paths, or ``(None, None)`` to drop it.++ Only symbol-layer edges are guarded; file-layer edges such as a module's+ ``co_changes`` with its docs pass as they are.+ """+ src = src or ""+ tgt = tgt or ""+ if not is_symbol_node(src) and not is_symbol_node(tgt):+ return (src or None, tgt or None)+ src_file, tgt_file = node_to_file(src), node_to_file(tgt)+ if not keep_projected_edge(src_file, tgt_file, edge_type, confidence):+ return (None, None)+ return (src_file, tgt_file)+++async def _projected_edges(session: Any, repo_id: str) -> list[tuple[str, str]]:+ """Every non-containment edge for *repo_id*, as a pair of file paths.++ One scan projected in Python: a symbol-layer prefix match cannot use the+ node-id index. Containment always projects to a self-loop.+ """+ res = await session.execute(+ select(+ GraphEdge.source_node_id,+ GraphEdge.target_node_id,+ GraphEdge.edge_type,+ GraphEdge.confidence,+ ).where(+ GraphEdge.repository_id == repo_id,+ GraphEdge.edge_type.notin_(CONTAINMENT_EDGE_TYPES),+ )+ )+ pairs: list[tuple[str, str]] = []+ for src, tgt, etype, conf in res.all():+ src_file, tgt_file = _projected_pair(src, tgt, etype, conf)+ if src_file and tgt_file:+ pairs.append((src_file, tgt_file))+ return pairs+++def _hub_degree_cutoff(degree: dict[str, int]) -> int:+ """Degree at which a candidate counts as a hub: ``max(floor, p99)`` of the set."""+ if not degree:+ return 1 << 30+ values = sorted(degree.values())+ idx = min(len(values) - 1, int(len(values) * _HUB_DEGREE_PERCENTILE))+ return max(_HUB_DEGREE_FLOOR, values[idx])+++def _seed_neighbors(+ pairs: list[tuple[str, str]], seed_set: set[str], existing: set[Any]+) -> tuple[set[str], dict[str, int]]:+ """Files one hop from a seed and not yet a candidate, plus every file's degree."""+ neighbors: set[str] = set()+ degree: dict[str, int] = {}+ for src_file, tgt_file in pairs:+ degree[src_file] = degree.get(src_file, 0) + 1+ degree[tgt_file] = degree.get(tgt_file, 0) + 1+ for mine, theirs in ((src_file, tgt_file), (tgt_file, src_file)):+ if mine in seed_set and theirs not in existing:+ neighbors.add(theirs)+ return neighbors, degree+++async def _neighbor_pages(+ session: Any, neighbors: set[str]+) -> tuple[list[Any], dict[str, float]]:+ """File-page rows of *neighbors*, and each neighbour's PageRank."""+ page_res = await session.execute(+ select(Page.target_path, Page.summary, Page.page_type).where(+ Page.target_path.in_(sorted(neighbors)),+ Page.page_type == "file_page",+ )+ )+ page_rows = list(page_res.all())++ pr_res = await session.execute(+ select(GraphNode.node_id, GraphNode.pagerank).where(+ GraphNode.node_id.in_(sorted(neighbors)),+ GraphNode.node_type == "file",+ )+ )+ pr_by_path = {row[0]: float(row[1] or 0.0) for row in pr_res.all()}+ return page_rows, pr_by_path+++def _drop_hubs(page_rows: list[Any], degree: dict[str, int]) -> list[Any]:+ """*page_rows* without hubs, or all of them when every row is a hub."""+ # Cutoff over the candidates on offer, not the whole repo.+ cutoff = _hub_degree_cutoff({row[0]: degree.get(row[0], 0) for row in page_rows})+ non_hub = [row for row in page_rows if degree.get(row[0], 0) <= cutoff]+ return non_hub or page_rows+++def _expansion_candidates(+ seeds: list[dict], page_rows: list[Any], pr_by_path: dict[str, float]+) -> list[dict]:+ """One damped hit per neighbour page, most central first."""+ strongest_parent = max(+ seeds, key=lambda hit: hit.get("score", 0.0), default={}+ )+ parent_score = strongest_parent.get("score", 0.0)+ confidence_factor = strongest_parent.get("_confidence_score_factor")+ candidates: list[dict] = []+ for path, summary, page_type in page_rows:+ candidate = {+ "page_id": f"file_page:{path}",+ "target_path": path,+ "title": f"File: {path}",+ "summary": summary or "",+ "snippet": (summary or "")[:200],+ "page_type": page_type or "file_page",+ "score": parent_score * _GRAPH_EXPAND_DAMPING,+ "_sources": {"graph_expand"},+ "_expanded_from": "graph",+ "_pagerank": pr_by_path.get(path, 0.0),+ }+ if confidence_factor is not None:+ candidate["_confidence_score_factor"] = confidence_factor+ candidates.append(candidate)++ # Path breaks PageRank ties so the pick does not follow database row order.+ candidates.sort(key=lambda c: (-c.get("_pagerank", 0.0), c["target_path"]))+ return candidates+++async def expand_via_graph(hits: list[dict], ctx: Any, repo_id: str) -> list[dict]:+ """Add up to ``_GRAPH_EXPAND_MAX_NEW`` one-hop neighbours of the top hits.++ Recovers the right file when retrieval landed next to it (a consumer+ instead of the implementation). Only neighbours with a file page that are+ not yet candidates join, marked ``_expanded_from`` with a damped score.+ """+ if not hits:+ return hits+ # Seeded from pages only, so a pageless file cannot change what pages add.+ seeds = [h for h in hits if h.get("page_type") != PAGELESS_FILE][:_GRAPH_EXPAND_TOP_N]+ seed_paths = [h.get("target_path") for h in seeds if h.get("target_path")]+ if not seed_paths:+ return hits+ existing = {h.get("target_path") for h in hits}++ async with get_session(ctx.session_factory) as session:+ # Every edge type, co-changes included, projected from the symbol layer+ # too so call edges reach the seeds.+ pairs = await per_index(+ session, repo_id, "projected_edges", lambda: _projected_edges(session, repo_id)+ )+ neighbors, degree = _seed_neighbors(pairs, set(seed_paths), existing)+ if not neighbors:+ return hits+ page_rows, pr_by_path = await _neighbor_pages(session, neighbors)++ if not page_rows:+ return hits++ page_rows = _drop_hubs(page_rows, degree)+ additions = _expansion_candidates(seeds, page_rows, pr_by_path)[:_GRAPH_EXPAND_MAX_NEW]+ if not additions:+ return hits++ combined = hits + additions+ combined.sort(key=lambda h: h["score"], reverse=True)+ return combineddiff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.pynew file mode 100644index 000000000..00926a8a8--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.py@@ -0,0 +1,143 @@+"""Hydrate hits with Page metadata, then bias them by PageRank."""++from __future__ import annotations++import logging+from typing import Any++from sqlalchemy import select++from repowise.core.persistence.database import get_session+from repowise.core.persistence.models import GraphNode, Page+from repowise.server.mcp_server._page_paths import PAGELESS_FILE, pageless_path++_log = logging.getLogger("repowise.mcp.answer")++# Capped multiplicative bias, [1.0, 1.3]: breaks ties, never beats a strong match.+_PAGERANK_BIAS_MAX = 0.3+++async def _page_meta_by_id(ctx: Any, page_ids: list[str]) -> dict[str, dict[str, str]]:+ """Path, summary, type and freshness of each page id that has a page row."""+ async with get_session(ctx.session_factory) as session:+ res = await session.execute(+ select(+ Page.id,+ Page.target_path,+ Page.summary,+ Page.page_type,+ Page.freshness_status,+ ).where(Page.id.in_(page_ids))+ )+ return {+ row[0]: {+ "target_path": row[1] or "",+ "summary": row[2] or "",+ "page_type": row[3] or "",+ "freshness": row[4] or "",+ }+ for row in res.all()+ }+++def _hit_meta(hit: dict, meta_by_id: dict[str, dict[str, str]]) -> dict[str, str] | None:+ """The page metadata behind *hit*, or None when there is nothing to cite."""+ meta = meta_by_id.get(hit["page_id"])+ if meta is None and hit.get("page_type") == PAGELESS_FILE:+ # A pageless file: its path is in its id, and it serves matched code.+ meta = {"target_path": pageless_path(hit["page_id"]), "page_type": PAGELESS_FILE}+ return meta+++def _apply_meta(hit: dict, meta: dict[str, str], scope: str | None) -> bool:+ """Copy *meta* onto *hit*; False when the hit is tombstoned or out of scope."""+ # Tombstoned pages describe deleted or renamed files.+ if meta.get("freshness") == "tombstone":+ return False+ target_path = meta.get("target_path", "")+ if scope and target_path and not target_path.startswith(scope):+ return False+ hit["target_path"] = target_path+ hit["summary"] = meta.get("summary", "")+ # The page table's type wins; retrievers can carry stale ones.+ hit["page_type"] = meta.get("page_type") or hit.get("page_type", "")+ return True+++async def hydrate_hits(hits: list[dict], ctx: Any, *, scope: str | None = None) -> list[dict]:+ """Attach path, summary and page type in place.++ Drops hits with no page, tombstoned pages and paths outside ``scope``.+ """+ if not hits:+ return hits+ meta_by_id = await _page_meta_by_id(ctx, [h["page_id"] for h in hits])++ out: list[dict] = []+ pageless = 0+ for h in hits:+ meta = _hit_meta(h, meta_by_id)+ if meta is None:+ # No page behind the id (a decision vector or store drift): nothing+ # to cite, and the later gates are keyed on a path it lacks.+ pageless += 1+ continue+ if _apply_meta(h, meta, scope):+ out.append(h)+ if pageless:+ _log.warning(+ "get_answer dropped %d of %d retrieved hit(s) with no page row behind them; "+ "an id in a retriever that the page table does not know is either a "+ "non-page vector or a three-store disagreement (repowise doctor --repair)",+ pageless,+ len(hits),+ )+ return out+++async def _file_pageranks(ctx: Any, paths: list[str]) -> dict[str, float]:+ """PageRank of each file node among *paths*."""+ async with get_session(ctx.session_factory) as session:+ res = await session.execute(+ select(GraphNode.node_id, GraphNode.pagerank).where(+ GraphNode.node_id.in_(paths),+ GraphNode.node_type == "file",+ )+ )+ return {row[0]: float(row[1] or 0.0) for row in res.all()}+++def _max_page_rank(hits: list[dict], pr_by_path: dict[str, float]) -> float:+ """The PageRank the bias normalises against."""+ # Over pages only, so a pageless file cannot change their bias.+ paged = {h.get("target_path") for h in hits if h.get("page_type") != PAGELESS_FILE}+ max_pr = max((pr for path, pr in pr_by_path.items() if path in paged), default=0.0)+ return max_pr or max(pr_by_path.values(), default=0.0)+++async def apply_pagerank_bias(hits: list[dict], ctx: Any) -> None:+ """Multiply each hit's score by a PageRank bias, then re-sort (in place).++ Normalised within the candidates, not the whole graph, so a famous but+ unrelated file gets no extra reward.+ """+ if not hits:+ return+ paths = [h.get("target_path") for h in hits if h.get("target_path")]+ if not paths:+ return+ pr_by_path = await _file_pageranks(ctx, paths)+ if not pr_by_path:+ return+ max_pr = _max_page_rank(hits, pr_by_path)+ if max_pr <= 0:+ return++ for h in hits:+ pr = pr_by_path.get(h.get("target_path"), 0.0)+ bias = 1.0 + _PAGERANK_BIAS_MAX * min(pr / max_pr, 1.0)+ h["_pagerank"] = pr+ h["_pagerank_bias"] = round(bias, 3)+ h["score"] = h.get("score", 0.0) * bias++ hits.sort(key=lambda h: h["score"], reverse=True)diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.pynew file mode 100644index 000000000..f262b392e--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.py@@ -0,0 +1,43 @@+"""Which retrieval legs ran for the current question, and how each ended."""++from __future__ import annotations++import contextvars++# How each leg ended for the current question. Legs fail soft, so without+# this a lexical-only answer looks like a whole one; ``embedder_live`` says an+# embedder exists, this says whether it was used.+_LEG_RECORD: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar(+ "repowise_retrieval_legs", default=None+)+++def _record_leg(leg: str, outcome: str) -> None:+ """Note how one retrieval leg ended, if anyone is collecting."""+ record = _LEG_RECORD.get()+ if record is not None:+ record[leg] = outcome+++def begin_leg_record() -> dict[str, str]:+ """Start collecting leg outcomes for one question. Returns the record.++ ``asyncio.gather`` copies the context, not the dict, so every leg task+ writes into this one record.+ """+ record: dict[str, str] = {}+ _LEG_RECORD.set(record)+ return record+++def retrieval_legs() -> dict[str, str]:+ """Leg outcomes for the last question, e.g. ``{"fts": "ok", "vector": "timeout"}``.++ Holds an ``embed`` key when embedding the question failed.+ """+ return dict(_LEG_RECORD.get() or {})+++def degraded_legs(legs: dict[str, str]) -> list[str]:+ """The legs that broke, sorted. ``keyless`` is configuration, not breakage."""+ return sorted(leg for leg, outcome in legs.items() if outcome not in ("ok", "keyless"))diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.pynew file mode 100644index 000000000..ef4b4b93d--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.py@@ -0,0 +1,35 @@+"""Noise demotion: decision records and test file pages sink below real hits."""++from __future__ import annotations++import re++from repowise.core.test_paths import is_test_path+from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES++# Same test-path rules as search_codebase's demotion, on the answer hit shape.+_TEST_QUERY_RE = re.compile(+ r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",+ re.IGNORECASE,+)+++def demote_noise_hits(hits: list[dict], question: str, *, is_why: bool) -> list[dict]:+ """Stable-partition noise below real hits before the top-5 cap; nothing is dropped.++ Decision records are noise unless the question is why-shaped, test files+ unless it is about tests. Test support such as ``conftest.py`` is not noise.+ """+ if not hits:+ return hits+ test_focused = bool(_TEST_QUERY_RE.search(question))++ def _is_noise(h: dict) -> bool:+ pt = h.get("page_type")+ return (pt == "decision_record" and not is_why) or (+ pt in FILE_ROW_TYPES and not test_focused and is_test_path(h.get("target_path") or "")+ )++ real = [h for h in hits if not _is_noise(h)]+ noise = [h for h in hits if _is_noise(h)]+ return real + noisediff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.pynew file mode 100644index 000000000..b7ed0c61d--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.py@@ -0,0 +1,215 @@+"""Parent-concept surfacing: a subsystem-shaped question leads with its concept page.++Retrieval ranks a subsystem's member files and child pages above the concept+page for the whole subsystem, because they match the question's noun more+tightly. Two signals pick the page, neither tuned to a repository: a tight+cluster of hits in one directory (structural), else a concept-restricted vector+search (semantic). Every other question keeps its ranking.+"""++from __future__ import annotations++import asyncio+import re+from typing import Any++from sqlalchemy import select++from repowise.core.persistence.database import get_session+from repowise.core.persistence.models import Page+from repowise.server.mcp_server._answer_pipeline.vectors import question_vector, vector_search+from repowise.server.mcp_server._helpers import vector_search_timeout_s++# High precision on purpose: each phrase is anchored to an overview intent,+# because fragments like "parts of" or "which module" fill ordinary questions.+_SUBSYSTEM_QUERY_RE = re.compile(+ r"\boverview of\b|\bgive me an overview\b|\bhigh[- ]level\b|"+ r"\bmain parts of\b|\bwhat (?:are|is) the (?:main )?(?:parts|pieces|components) of\b|"+ r"\barchitecture of\b|\bwalk me through\b|"+ r"\bhow (?:is|are|does) .+ (?:organi[sz]ed|structured|laid out|put together)\b|"+ r"\bwhat subsystem\b|\bwhich subsystem\b|"+ r"\bwhere would i (?:add|put)\b|\bwhere do i (?:add|put)\b",+ re.IGNORECASE,+)++# Strongest real hits (no decision records, no pathless pages) to cluster.+_PARENT_EXPAND_TOP_N = 8+# Hits a directory must directly hold to count as a tight cluster.+_PARENT_MIN_SHARE = 2+# Just past the top score: leads without reading as a dominant retrieval.+_PARENT_EXPAND_BOOST = 1.02+# Wide, since concept pages are a small minority of the vector store.+_CONCEPT_FETCH_LIMIT = 60+++def is_subsystem_query(question: str) -> bool:+ """True when the question asks about a subsystem as a whole."""+ return bool(_SUBSYSTEM_QUERY_RE.search(question or ""))+++def _common_ancestor(paths: set[str]) -> str:+ """Longest shared directory prefix of the given paths, segment-wise."""+ split = [p.split("/") for p in paths]+ common: list[str] = []+ for segs in zip(*split, strict=False):+ if len(set(segs)) == 1:+ common.append(segs[0])+ else:+ break+ return "/".join(common)+++async def _semantic_concept_paths(question: str, ctx: Any) -> list[str]:+ """Concept/layer page paths a concept-restricted vector search ranks first; [] on error."""+ vs = getattr(ctx, "vector_store", None)+ if vs is None:+ return []+ try:+ results = await asyncio.wait_for(+ vector_search(+ vs, question, _CONCEPT_FETCH_LIMIT, vector=await question_vector(ctx, question)+ ),+ timeout=vector_search_timeout_s(),+ )+ except Exception:+ return []+ out: list[str] = []+ for r in results:+ if getattr(r, "page_type", "") not in ("module_page", "layer_page"):+ continue+ tp = _result_target_path(r)+ if tp and tp not in out:+ out.append(tp)+ return out+++def _result_target_path(result: Any) -> str:+ """A vector hit's target path, read from its page id when it carries none."""+ tp = getattr(result, "target_path", "") or ""+ if not tp:+ pid = getattr(result, "page_id", "") or ""+ tp = pid.split(":", 1)[1] if ":" in pid else pid+ return tp+++def _cluster_parents(top: list[dict]) -> tuple[dict[str, int], dict[str, set[str]]]:+ """Hits directly in each directory, and the hits under each ancestor."""+ imm_count: dict[str, int] = {}+ covers: dict[str, set[str]] = {}+ for h in top:+ tp = h["target_path"].rstrip("/")+ parent = tp.rsplit("/", 1)[0] if "/" in tp else ""+ if parent:+ imm_count[parent] = imm_count.get(parent, 0) + 1+ anc = parent+ while anc:+ covers.setdefault(anc, set()).add(tp)+ anc = anc.rsplit("/", 1)[0] if "/" in anc else ""+ return imm_count, covers+++async def _concept_pages(ctx: Any, candidates: set[str]) -> list[Any]:+ """Concept/layer page rows whose target path is one of *candidates*."""+ async with get_session(ctx.session_factory) as session:+ return (+ await session.execute(+ select(Page.target_path, Page.title, Page.summary, Page.page_type).where(+ Page.target_path.in_(candidates),+ Page.page_type.in_(("module_page", "layer_page")),+ )+ )+ ).all()+++def _tight_cluster_page(+ tight_pages: set[str],+ by_path: dict[str, Any],+ imm_count: dict[str, int],+ covers: dict[str, set[str]],+) -> Any:+ """The tightest cluster's page; sibling clusters roll up to their common ancestor."""+ rollup = _common_ancestor(tight_pages)+ if len(tight_pages) > 1 and rollup in by_path and rollup in covers:+ return by_path[rollup]+ best_tp = max(tight_pages, key=lambda d: (imm_count[d], d.count("/")))+ return by_path[best_tp]+++def _semantic_or_cover_page(+ semantic_paths: list[str], by_path: dict[str, Any], covers: dict[str, set[str]]+) -> Any:+ """The semantically closest concept page, else the deepest covering ancestor."""+ winner = next((by_path[tp] for tp in semantic_paths if tp in by_path), None)+ if winner is None:+ cov_pages = [a for a in covers if a in by_path]+ if cov_pages:+ winner = by_path[max(cov_pages, key=lambda a: (a.count("/"), len(covers[a])))]+ return winner+++def _lead_with_parent(hits: list[dict], winner: Any) -> list[dict]:+ """Promote *winner* to lead if already retrieved, else inject it as the lead."""+ best_tp, best_title, best_summary, best_pt = winner+ top_score = max((h.get("score", 0.0) for h in hits), default=0.0) or 1.0+ lead_score = top_score * _PARENT_EXPAND_BOOST++ retrieved = next((h for h in hits if h.get("target_path") == best_tp), None)+ if retrieved is not None:+ if retrieved.get("score", 0.0) < lead_score:+ retrieved["score"] = lead_score+ src = retrieved.get("_sources")+ if isinstance(src, set):+ src.add("parent_promote")+ hits.sort(key=lambda x: x.get("score", 0.0), reverse=True)+ return hits++ parent_hit = {+ "page_id": f"{best_pt}:{best_tp}",+ "target_path": best_tp,+ "title": best_title or f"Overview: {best_tp}",+ "summary": best_summary or "",+ "snippet": (best_summary or "")[:200],+ "page_type": best_pt or "module_page",+ "score": lead_score,+ "_sources": {"parent_expand"},+ "_expanded_from": "parent",+ }+ combined = [parent_hit, *hits]+ combined.sort(key=lambda h: h["score"], reverse=True)+ return combined+++async def expand_via_parent_page(hits: list[dict], question: str, ctx: Any) -> list[dict]:+ """Lead a subsystem-shaped question with the concept page for its subsystem.++ A tight structural cluster wins over the semantic guess. The page is+ promoted in place when already retrieved, else injected as the lead.+ """+ if not hits or not is_subsystem_query(question):+ return hits+ top = [h for h in hits if h.get("target_path") and h.get("page_type") != "decision_record"][+ :_PARENT_EXPAND_TOP_N+ ]+ if not top:+ return hits++ imm_count, covers = _cluster_parents(top)+ tight_clusters = {d for d, c in imm_count.items() if c >= _PARENT_MIN_SHARE}+ semantic_paths = await _semantic_concept_paths(question, ctx)+ candidates = set(covers) | set(imm_count) | set(semantic_paths)+ if not candidates:+ return hits++ rows = await _concept_pages(ctx, candidates)+ if not rows:+ return hits+ by_path = {r[0]: r for r in rows}++ tight_pages = tight_clusters & by_path.keys()+ if tight_pages:+ winner = _tight_cluster_page(tight_pages, by_path, imm_count, covers)+ else:+ winner = _semantic_or_cover_page(semantic_paths, by_path, covers)+ if winner is None:+ return hits+ return _lead_with_parent(hits, winner)diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.pynew file mode 100644index 000000000..b50ddff5d--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.py@@ -0,0 +1,88 @@+"""The question's embedding, computed once per call, and the shared vector read."""++from __future__ import annotations++import asyncio+import logging+from collections import OrderedDict+from typing import Any++from repowise.core.providers.embedding import store_has_semantic_vectors+from repowise.server.mcp_server._answer_pipeline.leg_record import _record_leg+from repowise.server.mcp_server._helpers import _EMBED_TIMEOUT_ENV, embed_timeout_s++_log = logging.getLogger("repowise.mcp.answer")++# One embedding per (store, question), shared by every stage that needs it.+# The key holds the store in its value so a recycled id cannot match. A few+# entries so concurrent questions do not evict each other.+_QUESTION_VECTOR_CACHE_MAX = 4+_QUESTION_VECTORS: OrderedDict[tuple[int, str], tuple[Any, list[float]]] = OrderedDict()+++async def question_vector(ctx: Any, question: str) -> list[float] | None:+ """The question's embedding, cached per store and question.++ None when there is no store or embedder, or the embed failed; callers then+ search by text, which costs an embed but is still correct.+ """+ store = getattr(ctx, "vector_store", None)+ if store is None or not question:+ return None+ if not store_has_semantic_vectors(store):+ return None++ key = (id(store), question)+ cached = _QUESTION_VECTORS.get(key)+ if cached is not None:+ _QUESTION_VECTORS.move_to_end(key)+ return cached[1]++ timeout_s = embed_timeout_s()+ try:+ vectors = await asyncio.wait_for(store.embed_texts([question]), timeout=timeout_s)+ except TimeoutError:+ # Recorded so a lexical-only answer is visible as one.+ _record_leg("embed", "timeout")+ _log.warning(+ "get_answer could not embed the question within %.1fs; retrieval "+ "continues without a question vector. Raise it with %s=<seconds>.",+ timeout_s,+ _EMBED_TIMEOUT_ENV,+ )+ return None+ except Exception:+ _record_leg("embed", "error")+ _log.warning(+ "get_answer could not embed the question up front; each retrieval "+ "stage will embed it again",+ exc_info=True,+ )+ return None+ if not vectors:+ # Backend without its own embedder: each stage embeds the text itself.+ _log.debug("Vector store cannot embed directly; per-stage embedding stands")+ return None++ vector = [float(v) for v in vectors[0]]+ _QUESTION_VECTORS[key] = (store, vector)+ while len(_QUESTION_VECTORS) > _QUESTION_VECTOR_CACHE_MAX:+ _QUESTION_VECTORS.popitem(last=False)+ return vector+++async def vector_search(+ store: Any, question: str, limit: int, *, vector: list[float] | None+) -> list[Any]:+ """Nearest pages to *question*, by *vector* when the backend supports it.++ [] on a keyless store; checked here because every vector read in the+ pipeline goes through this, and the text fallback would embed.+ """+ if not store_has_semantic_vectors(store):+ return []+ if vector is not None:+ by_vector = await store.search_by_vector(vector, limit=limit)+ if by_vector is not None:+ return by_vector+ return await store.search(question, limit=limit)diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pyindex 748c4929e..791895736 100644--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py@@ -7,6 +7,7 @@ import contextlibimport loggingimport refrom collections.abc import Container, Sequence+from dataclasses import dataclassfrom typing import Anyfrom sqlalchemy import select@@ -1053,6 +1054,246 @@ def _missing_named_symbols(return missing+@dataclass(frozen=True)+class _SearchPlan:+ """One structured search: the caller's arguments plus the derived query."""++ query: str+ limit: int+ page_type: str | None+ kind: str | None+ symbol_kind: str | None+ mode: str+ names: Container[str] | None+ canonical_symbol: tuple[Any, str] | None+ symbol_query: str+ fetch: int+ multi: bool+++@dataclass+class _Pools:+ """Each row kind merged and ranked across the repos searched."""++ contexts: list[Any]+ symbols: list[dict]+ spare: list[dict]+ files: list[dict]+ indexed_paths: list[str]+ concepts: list[dict]+++def _symbol_query_for(+ query: str, mode: str, names: Container[str] | None, canonical_symbol: Any+) -> str:+ """The text the symbol scorer ranks on."""+ # Scored on the identifiers, not the prose: words like "is" would match+ # ``is_ci`` and bury the symbol the question names.+ symbol_query = query+ if canonical_symbol:+ symbol_query = canonical_symbol[1]+ if mode == "hybrid":+ _idents = _embedded_identifiers(query, names)+ # No identifier: content words, else the raw text.+ symbol_query = (+ " ".join(_idents) if _idents else " ".join(content_terms(query)) or query+ )+ return symbol_query+++def _symbol_fetch(mode: str, limit: int) -> int:+ """How many symbol rows each repo fetches."""+ # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every+ # candidate so ``fuzzy_omitted`` can count what an exact hit hides.+ fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit+ if mode == "symbol":+ fetch = _MAX_CANDIDATES+ return fetch+++async def _search_one_repo(+ ctx: Any, plan: _SearchPlan+) -> tuple[list[dict], list[dict], list[str], list[dict]]:+ """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""+ s: list[dict] = []+ f: list[dict] = []+ paths: list[str] = []+ c: list[dict] = []+ if plan.mode in ("symbol", "hybrid"):+ s = await search_symbols_single(+ ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind+ )+ if plan.mode == "path":+ f = await search_paths_single(ctx, plan.query, plan.limit)+ async with get_session(ctx.session_factory) as session:+ paths = (await file_path_index(session, (await _get_repo(session)).id)).paths+ if plan.mode == "hybrid":+ c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)+ for item in c:+ item["type"] = "page"+ await attach_hit_symbols(ctx, plan.query, c)+ for rows in (s, f, c):+ _tag_repo(rows, ctx, plan.multi)+ return s, f, paths, c+++def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:+ """Merge each repo's rows into one ranked pool per row kind."""+ parts = [part for _, part in answered]+ return _Pools(+ contexts=[ctx for ctx, _ in answered],+ symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),+ # Rows past each repo's limit only backfill slots that collapsing frees.+ spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),+ files=merge_ranked((p[1] for p in parts), _neg_score("score")),+ indexed_paths=[path for p in parts for path in p[2]],+ concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),+ )+++def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:+ """Keep exact name and path matches ahead of their fuzzy neighbours."""+ if plan.mode == "symbol":+ pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)+ elif plan.mode == "hybrid" and candidates:+ pools.symbols = _protect_named_symbols(candidates, pools.symbols)+ if plan.mode == "path":+ pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)+++def _symbol_mode_results(+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool+) -> tuple[list[dict], int]:+ """Symbol mode rows, and how many fuzzy names an exact hit hides."""+ # An exact hit is the answer; its fuzzy neighbours are only counted.+ fuzzy_omitted = 0+ pool = pools.symbols + pools.spare+ named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []+ if named:+ canonical_symbol = plan.canonical_symbol+ ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()+ shown = {id(s) for s in named}+ fuzzy_omitted = sum(+ 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()+ )+ return (named or pools.symbols)[: plan.limit], fuzzy_omitted+++def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:+ """Hybrid rows: symbol matches interleaved with concept pages for new files."""+ window = _hybrid_window(+ plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names+ )+ # One row per file; freed slots fill with the next pages, then symbols.+ served = {id(item) for item in window}+ rest = [+ item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served+ ]+ results = _collapse_by_file(window + rest)[: plan.limit]+ # Non-file pages leave the window unrefilled unless the caller asked for+ # pages by type or kind="doc".+ if not plan.page_type and plan.kind != "doc":+ results = [item for item in results if hit_file_path(item)]+ return results+++def _select_results(+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool+) -> tuple[list[dict], int]:+ """The served rows for the plan's mode, and the symbol-mode fuzzy count."""+ if plan.mode == "symbol":+ return _symbol_mode_results(plan, pools, candidates, exact)+ if plan.mode == "path":+ return pools.files[: plan.limit], 0+ return _hybrid_mode_results(plan, pools, exact), 0+++async def _single_repository(contexts: list[Any], multi: bool) -> Any:+ """The repository row a single-repo search ran against, else None."""+ if multi or not contexts:+ return None+ async with get_session(contexts[0].session_factory) as session:+ return await _get_repo(session)+++def _exact_match_note(+ candidates: list[str],+ exact: bool,+ missing: list[str],+ fuzzy_omitted: int,+ has_results: bool,+ grep_hint: str | None,+) -> str | None:+ """The note qualifying an identifier query's results, if one is owed."""+ if missing:+ shown = ", ".join(repr(c) for c in missing[:3])+ # An empty window's grep_hint already carries the sweep advice.+ pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""+ sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT+ return (+ f"No indexed symbol is named {shown}, so no symbol is returned "+ f"for it.{pages} Recheck the spelling, or search a shorter part "+ f"of the name.{sweep}"+ )+ if not exact:+ shown = ", ".join(repr(c) for c in candidates[:3])+ return (+ f"No indexed symbol exactly matches {shown}. The results are "+ "fuzzy neighbours ranked by token overlap — confirm a hit names "+ "what you meant before relying on it. If you expected an exact "+ "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT+ )+ if fuzzy_omitted:+ return (+ f"{fuzzy_omitted} other symbols contain this name; search a "+ "longer or partial name to list them."+ )+ return None+++def _match_named(+ plan: _SearchPlan, pools: _Pools, candidates: list[str]+) -> tuple[bool, list[str]]:+ """Whether a returned symbol names the query exactly, and the names none does.++ Missing names leave only exact symbols in *pools*.+ """+ exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False+ missing = _missing_named_symbols(+ candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts+ )+ if missing:+ pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]+ pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]+ return exact, missing+++def _attach_match_notes(+ response: dict,+ candidates: list[str],+ exact: bool,+ missing: list[str],+ grep_hint: str | None,+) -> None:+ """Exact-match flag, its note, and the grep fallback for an empty window."""+ results = response["results"]+ # Fuzzy neighbours of a missing name look authoritative, so say so.+ if candidates:+ response["exact_match"] = exact and not missing+ note = _exact_match_note(+ candidates,+ exact,+ missing,+ response.get("fuzzy_omitted", 0),+ bool(results),+ grep_hint,+ )+ if note:+ response["note"] = note+ if grep_hint and not results:+ response["grep_hint"] = grep_hint++async def _structured_search(query: str,limit: int,@@ -1065,128 +1306,36 @@ async def _structured_search(names: Container[str] | None = None,fed: Federation | None = None,) -> dict:- """Run symbol / path / hybrid search and shape the response.+ """Run symbol / path / hybrid search, across repos when federated, and shape the response.- Honours ``repo="all"`` (federates across contexts, then re-ranks by score),- per-repo ``exclude_patterns`` and tombstones (enforced inside the- single-repo helpers). The grep_hint is attached only as a fallback — when- the structural index produced nothing, the agent still has a path forward.+ ``grep_hint`` is attached only when nothing was found."""multi = fed is not None and fed.multi-- # A hybrid query is prose wrapped around an identifier ("where is X- # defined"). The symbol scorer ranks on token overlap, so handing it the- # raw prose lets stopword-ish tokens ("is" -> is_ci, "filter" ->- # FilterRegistry) outrank the identifier the question is actually about,- # which then never reaches _has_exact_symbol and the response claims the- # symbol is unindexed. Score symbols on the extracted identifiers instead.- symbol_query = querycanonical_symbol = _canonical_symbol_query(query)- if canonical_symbol:- symbol_query = canonical_symbol[1]- if mode == "hybrid":- _idents = _embedded_identifiers(query, names)- # Without an identifier, score on the content words so stopwords such- # as "on" can't match symbols like ``test_basic_auth_on_session``. A- # query made only of stopwords keeps the raw text.- symbol_query = (- " ".join(_idents) if _idents else " ".join(content_terms(query)) or query- )-- # Only the hybrid window collapses symbol rows, so only it over-fetches- # them; path hits are file pages, already one row per file. Symbol mode- # takes every candidate so ``fuzzy_omitted`` counts the names an exact hit hides.- fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit- if mode == "symbol":- fetch = _MAX_CANDIDATES+ plan = _SearchPlan(+ query=query,+ limit=limit,+ page_type=page_type,+ kind=kind,+ symbol_kind=symbol_kind,+ mode=mode,+ names=names,+ canonical_symbol=canonical_symbol,+ symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),+ fetch=_symbol_fetch(mode, limit),+ multi=multi,+ )async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:- s: list[dict] = []- f: list[dict] = []- paths: list[str] = []- c: list[dict] = []- if mode in ("symbol", "hybrid"):- s = await search_symbols_single(- ctx, symbol_query, fetch, symbol_kind=symbol_kind, kind=kind- )- if mode == "path":- f = await search_paths_single(ctx, query, limit)- async with get_session(ctx.session_factory) as session:- paths = (await file_path_index(session, (await _get_repo(session)).id)).paths- if mode == "hybrid":- c = await _search_single_repo(ctx, query, limit, page_type, kind)- for item in c:- item["type"] = "page"- await attach_hit_symbols(ctx, query, c)- for rows in (s, f, c):- _tag_repo(rows, ctx, multi)- return s, f, paths, c-- answered = await _per_repo(repo, fed, _one_repo)- contexts = [ctx for ctx, _ in answered]- parts = [part for _, part in answered]- # Symbol rows past each repo's ``limit``: they only backfill slots that- # collapsing same-file rows frees, and never steer the exact-match signal.- symbols = merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit)- spare = merge_ranked((p[0][limit:] for p in parts), _neg_score("score"))- files = merge_ranked((p[1] for p in parts), _neg_score("score"))- indexed_paths = [path for p in parts for path in p[2]]- # Re-ranked by relevance so a strong page in repo B isn't buried under- # repo A's weak ones.- concepts = merge_ranked((p[3] for p in parts), _neg_score("relevance_score"))+ return await _search_one_repo(ctx, plan)++ pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)candidates = _identifier_candidates(query, mode, names)- if mode == "symbol":- symbols = _protect_exact_symbols(query, symbols)- elif mode == "hybrid" and candidates:- symbols = _protect_named_symbols(candidates, symbols)- if mode == "path":- files = _protect_exact_paths(query, files, indexed_paths)-- # Whether any returned symbol matches the query's identifier(s) exactly.- # Computed once here so the hybrid interleave and the exact-match note below- # agree on the same signal.- exact = _has_exact_symbol(candidates, symbols) if candidates else False- missing = _missing_named_symbols(candidates, symbols, bool(canonical_symbol), concepts)- if missing:- symbols = [s for s in symbols if _has_exact_symbol(candidates, [s])]- spare = [s for s in spare if _has_exact_symbol(candidates, [s])]+ _protect_exact(plan, pools, candidates)+ exact, missing = _match_named(plan, pools, candidates)- fuzzy_omitted = 0- if mode == "symbol":- # An exact hit is the answer; its fuzzy neighbours only cost bytes.- # Summed over repos when federated, and a floor past the candidate cap.- pool = symbols + spare- named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []- if named:- ident = (canonical_symbol[1] if canonical_symbol else query).strip().lower()- shown = {id(s) for s in named}- fuzzy_omitted = sum(- 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()- )- results = (named or symbols)[:limit]- elif mode == "path":- results = files[:limit]- else: # hybrid: interleave symbol matches and concept pages for new files- window = _hybrid_window(query, symbols, concepts, limit, exact, names)- # One row per file, so ``limit`` buys distinct files (mode "symbol"- # stays row-per-symbol: overloads there are the answer). Collapsing- # keeps the window's order and frees the slots same-file rows took;- # the next pages, then the next symbols, fill them.- served = {id(item) for item in window}- rest = [item for item in concepts + symbols + spare if id(item) not in served]- results = _collapse_by_file(window + rest)[:limit]- # A code-location query wants files to open, so a module, onboarding- # or decision page is dropped from the window. Its slot is not- # refilled: on the retrieval guard a refill bought +0.01 coverage at- # limit 5 for -0.01 precision at limit 10. Kept when the caller asked- # for pages by type or kind="doc".- if not page_type and kind != "doc":- results = [item for item in results if hit_file_path(item)]-- repository = None- if not multi and contexts:- async with get_session(contexts[0].session_factory) as session:- repository = await _get_repo(session)+ results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)+ repository = await _single_repository(pools.contexts, multi)response: dict = {"results": results,@@ -1195,59 +1344,16 @@ async def _structured_search(}if fuzzy_omitted:response["fuzzy_omitted"] = fuzzy_omitted- # Symbols first, then everything else the window holds: in symbol and- # hybrid modes the ranked pool leads with symbol hits, and those are the- # entries most likely to collapse onto one another (several symbols of one- # file). Deduping them is the point.- #- # Bound to its own name, NOT to ``candidates``: that one holds the query's- # identifiers, which the exact-match note below quotes. Rebinding it here- # made the note quote file paths as if they were the identifiers asked for,- # and — worse — made its gate true for any query with results at all, so a- # prose query that names no identifier was told no symbol matched it.+ # Not ``candidates``: that name holds the query's identifiers the note quotes.if file_cands := file_candidates(results, limit=limit):response["candidates"] = file_cands- # Exact-match honesty: an identifier-shaped query whose target names no- # indexed symbol still returns fuzzy neighbours. Say so, or the agent- # anchors on a wrong hit that looks authoritative (their Alamofire- # 44-overload read-spiral). Emit the boolean either way; a note only when- # there is no exact hit to distinguish from the fuzz. ``candidates`` /- # ``exact`` were computed above so ordering and this note stay consistent.- if candidates:- response["exact_match"] = exact and not missing- if missing:- shown = ", ".join(repr(c) for c in missing[:3])- # An empty window has no page to qualify, and its grep_hint- # already carries the sweep advice.- pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if results else ""- sweep = "" if (grep_hint and not results) else " " + EXHAUSTIVE_SWEEP_HINT- response["note"] = (- f"No indexed symbol is named {shown}, so no symbol is returned "- f"for it.{pages} Recheck the spelling, or search a shorter part "- f"of the name.{sweep}"- )- elif not exact:- shown = ", ".join(repr(c) for c in candidates[:3])- response["note"] = (- f"No indexed symbol exactly matches {shown}. The results are "- "fuzzy neighbours ranked by token overlap — confirm a hit names "- "what you meant before relying on it. If you expected an exact "- "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT- )- elif fuzzy_omitted:- response["note"] = (- f"{fuzzy_omitted} other symbols contain this name; search a "- "longer or partial name to list them."- )- if grep_hint and not results:- response["grep_hint"] = grep_hint+ _attach_match_notes(response, candidates, exact, missing, grep_hint)if fed is not None:fed.annotate(response, results)- # Last, so nothing above has to know the field is on its way out. Paths- # first, so a page whose target_path is dropped keeps its page_id.+ # Last: slims fields the steps above still read._slim_served_rows(results,- {(ctx.alias if multi else None): _page_language(ctx) for ctx in contexts},+ {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},)_serve_snippets(results)return responsediff --git a/tests/unit/server/mcp/test_embed_timeout.py b/tests/unit/server/mcp/test_embed_timeout.pyindex 1bffb4832..0a6873b0c 100644--- a/tests/unit/server/mcp/test_embed_timeout.py+++ b/tests/unit/server/mcp/test_embed_timeout.py@@ -81,7 +81,9 @@ def test_no_hardcoded_eight_second_embed_budget() -> None:"""Pins the fix: the embed budget is resolved live, not baked in at import."""from pathlib import Path- text = Path(pipeline.__file__).read_text(encoding="utf-8")+ text = "".join(+ p.read_text(encoding="utf-8") for p in sorted(Path(pipeline.__file__).parent.glob("*.py"))+ )assert "_EMBED_TIMEOUT_S = 8.0" not in textassert "_EMBED_TIMEOUT_S =" not in text # no hardcoded module-level constant at alldiff --git a/tests/unit/server/mcp/test_vector_search_timeout.py b/tests/unit/server/mcp/test_vector_search_timeout.pyindex 512e01c51..c29650b2d 100644--- a/tests/unit/server/mcp/test_vector_search_timeout.py+++ b/tests/unit/server/mcp/test_vector_search_timeout.py@@ -99,6 +99,6 @@ def test_no_hardcoded_eight_second_vector_budget() -> None:from pathlib import Pathroot = Path(tool_search.__file__).parent- for name in ("tool_search.py", "_answer_pipeline.py"):- text = (root / name).read_text(encoding="utf-8")- assert "timeout=8.0" not in text, f"{name} still hardcodes the old vector budget"+ for path in (root / "tool_search.py", *sorted((root / "_answer_pipeline").glob("*.py"))):+ text = path.read_text(encoding="utf-8")+ assert "timeout=8.0" not in text, f"{path.name} still hardcodes the old vector budget"