Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(answer): answer retrieval pipeline as a stage package; split _structured_search (#4010)

Raghav Chamadiyaa09e107127c7
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py
deleted file mode 100644
index 2130dc808..000000000
--- a/packages/server/src/repowise/server/mcp_server/_answer_pipeline.py
+++ /dev/null
@@ -1,1251 +0,0 @@
-"""Retrieval pipeline for ``get_answer``.
-
-This module owns everything that turns a developer question into a ranked
-list of candidate wiki hits — but emphatically not the LLM step, the cache,
-or the response shape (those live in ``tool_answer``). Separation of concerns
-lets us iterate on retrieval quality without rereading the orchestrator and
-vice versa.
-
-Pipeline (each stage is a pure function over hit dicts):
-
- 1. ``hybrid_retrieve`` : FTS, vector store and the structural symbol
- index in parallel, merged via Reciprocal
- Rank Fusion. Single retrieval modes
- systematically miss either token matches
- (vectors drift) or conceptual matches (FTS
- is literal), and both page-shaped modes miss
- anything a generated file page does not
- spell out: its public-symbol table is all
- they can index, so a private helper or a
- local name is invisible to them. The symbol
- leg covers that third class, for the cost of
- one more coroutine.
- 2. ``hydrate_hits`` — attach target_path, summary, page_type from
- the Page table to each hit.
- 3. ``apply_pagerank_bias`` — multiply scores by a damped PageRank factor
- so architecturally central files outrank
- peripheral ones at similar retrieval score.
- This is what rescues "how does X work"
- questions from peripheral consumers of X.
- 4. ``expand_via_graph`` — for top-N hits, walk 1 hop through the graph
- and pull in neighbors that have a wiki
- page. Rescues near-misses where retrieval
- landed in the right module but on a wrong
- file (consumer vs. orchestrator). Reference
- and co-change edges both qualify; see the
- note on the queries for why.
-
-Stages downstream of this module (term coverage, intersection boost, domain
-penalty) live in ``tool_answer`` for now — they're tightly coupled to the
-existing question-aware symbol promotion code and not worth duplicating.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import contextvars
-import logging
-import re
-from collections import OrderedDict
-from typing import Any
-
-from sqlalchemy import select
-
-from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX
-from repowise.core.ingestion.models import CONTAINMENT_EDGE_TYPES
-from repowise.core.persistence.database import get_session
-from repowise.core.persistence.models import GraphEdge, GraphNode, Page
-from repowise.core.providers.embedding import store_has_semantic_vectors
-from repowise.core.test_paths import is_test_path
-from repowise.server.mcp_server._graph_files import (
- is_symbol_node,
- keep_projected_edge,
- node_to_file,
- per_index,
-)
-from repowise.server.mcp_server._helpers import (
- _EMBED_TIMEOUT_ENV,
- _VECTOR_TIMEOUT_ENV,
- embed_timeout_s,
- vector_search_timeout_s,
-)
-from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES, PAGELESS_FILE, pageless_path
-from repowise.server.mcp_server._prose_symbols import filename_backed_pages, symbol_backed_pages
-
-_log = logging.getLogger("repowise.mcp.answer")
-
-# How many candidates each retriever fetches before merging. Both modes
-# tend to put the right answer in their top ~10, so 15 gives RRF room to
-# resolve ties without dragging weak tail hits into the merge.
-_RETRIEVAL_FETCH_LIMIT = 15
-
-# Page-id namespace decision vectors live under in the shared page store. Read
-# from the module that writes them so the two cannot drift apart.
-_DECISION_PREFIX = DECISION_VECTOR_PREFIX
-
-# Decision records share the page vector store under this page-id namespace, so
-# a page fetch has to ask for more rows than it needs and drop them. Over-fetch
-# rather than post-filter a fixed window: a query whose nearest neighbours are
-# all decisions would otherwise return a candidate set short by however many
-# happened to rank high, which reads downstream as thin retrieval rather than as
-# a filtered one. The margin is generous because decision rows cluster on the
-# question-shaped text that also retrieves well.
-_DECISION_OVERFETCH = 15
-
-# RRF constant. The standard k=60 from the original RRF paper — large enough
-# that rank-1 (1/61) and rank-2 (1/62) are close, small enough that rank-10
-# (1/70) still contributes meaningfully when a hit only shows up in one mode.
-_RRF_K = 60
-
-# Multiplier applied to RRF scores so they land in roughly the same numeric
-# range as the BM25 raw scores the downstream gates/thresholds (dominance
-# ratio, high-confidence score floor, absolute-gap branch) were tuned for.
-#
-# Top-1 RRF with both modes hitting at rank 0 is 1/60 + 1/60 ≈ 0.033.
-# Scaling by 180 puts it at ~6, matching the upper end of observed BM25
-# scores on this corpus. Pure ordering preservation — never changes
-# *which* hit ranks where, only the absolute numbers the gates compare.
-_RRF_SCORE_SCALE = 180.0
-
-# Cap how many extra files graph expansion can add. Without a cap, a hub
-# file (many importers) would flood the candidate set and dilute the LLM's
-# context budget on tangential neighbors.
-_GRAPH_EXPAND_TOP_N = 2
-_GRAPH_EXPAND_MAX_NEW = 3
-
-# Degree above which a neighbour is treated as a hub and dropped from the
-# expansion set. A file that half the repo imports is a near-neighbour of
-# everything, so it says nothing about *this* question; it is also exactly the
-# file PageRank ranks first, which is why the ordering below needs a guard
-# rather than trusting centrality on its own. Floor plus percentile, so a small
-# repo where every file has a handful of edges excludes nothing.
-_HUB_DEGREE_FLOOR = 50
-_HUB_DEGREE_PERCENTILE = 0.99
-
-# PageRank bias is multiplicative and capped. We don't want a marginally
-# more central file to outrank a strong text match — only to break ties.
-# Empirically PageRank values on this corpus span ~0 to ~0.01; we normalise
-# to the max in the candidate set and scale to a [1.0, 1.3] multiplier.
-_PAGERANK_BIAS_MAX = 0.3
-
-# Damping factor for graph-expanded hits. They didn't surface in retrieval,
-# so we trust them less than direct hits — but enough to outrank the bottom
-# of the top-5 if the parent was strong. 0.7 keeps a strong parent's child
-# (e.g. parent at 4.5, expanded child at 3.15) competitive with a real
-# rank-3/4 hit (~3.0-3.5).
-_GRAPH_EXPAND_DAMPING = 0.7
-
-# Budget for embedding the question is resolved live by embed_timeout_s()
-# (see _helpers.py) — REPOWISE_EMBED_TIMEOUT_S, falling back to the 8s a warm
-# hosted endpoint needs. The searches that used to embed inline were bounded
-# at that same 8s including the embed, so the round-trip keeps that ceiling
-# now that it happens on its own.
-
-# Which retrieval legs actually ran for the current question (finding A18).
-#
-# Every leg here is best-effort by design: a slow vector store must not be able
-# to block an answer. The defect was that the fallback was *silent*. An embed
-# that times out returns no vector, retrieval quietly continues lexical-only,
-# and every health signal still reports green — ``embedder_live`` included,
-# because a configured embedder is live whether or not this particular call
-# beat the clock. Five queries in one bake-off run were answered without their
-# vector leg and nothing in the response, the logs' structured fields, or the
-# per-cell record said so.
-#
-# So the leg outcome travels with the answer. ``embedder_live`` says the
-# embedder exists; this says whether it was used. Those are different claims
-# and only one of them is about the answer the caller is holding.
-_LEG_RECORD: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar(
- "repowise_retrieval_legs", default=None
-)
-
-
-def _record_leg(leg: str, outcome: str) -> None:
- """Note how one retrieval leg ended, if anyone is collecting."""
- record = _LEG_RECORD.get()
- if record is not None:
- record[leg] = outcome
-
-
-def begin_leg_record() -> dict[str, str]:
- """Start collecting leg outcomes for one question. Returns the record.
-
- The legs run under ``asyncio.gather``, which copies the context into each
- task. Copying rebinds names, not objects, so the tasks all mutate this one
- dict and the caller sees what they wrote.
- """
- record: dict[str, str] = {}
- _LEG_RECORD.set(record)
- return record
-
-
-def retrieval_legs() -> dict[str, str]:
- """How each retrieval leg ended for the question just answered.
-
- ``{"fts": "ok", "vector": "timeout", "symbol": "ok"}`` and so on, plus an
- ``embed`` key when embedding the question is what failed. Empty before any
- retrieval has run.
- """
- return dict(_LEG_RECORD.get() or {})
-
-
-def degraded_legs(legs: dict[str, str]) -> list[str]:
- """The legs that *broke*, named. Empty when retrieval was whole.
-
- ``keyless`` is not degradation and is deliberately excluded. A keyless index
- has no semantic vectors by construction, so its vector leg is permanently
- absent rather than transiently broken, and naming it here would put a
- failure marker on every answer the mode ever produces. The configuration
- itself is reported once per response by ``_meta`` instead.
- """
- return sorted(leg for leg, outcome in legs.items() if outcome not in ("ok", "keyless"))
-
-# Third retrieval leg: the structural symbol index. FTS and the vector store
-# both read the generated wiki page, which by construction carries an overview
-# sentence, the *public* symbol table and dependency paths, with no function
-# bodies, no private helpers, no local names. A question about something a page
-# does not spell out therefore has nothing to match on, however well it is
-# phrased. The symbol index does carry those names, and it is keyed on the
-# words a symbol is built from, so it recovers exactly that class of miss.
-#
-# How many symbols the leg ranks, and how many distinct files they may
-# contribute. Deliberately smaller than the other two legs' 15: this leg is
-# there to add pool members the page legs cannot see, not to outvote them.
-_SYMBOL_LEG_FETCH_LIMIT = 20
-_SYMBOL_LEG_MAX_PAGES = 8
-
-# The symbol leg's own RRF constant, larger than the page legs' so its
-# contribution is smaller at every rank. Fusing it at k=60 like the others gave
-# one lexical symbol match the same weight as a page two independent retrievers
-# ranked first, which is not what a name match is worth: measured on the
-# 99-question eval it pushed the correct ``distill/skeleton.py`` out of the
-# served five in favour of a same-named React component. At k=180 the leg can
-# lift a file the page retrievers already liked and can add one they never saw,
-# but it cannot outvote them.
-_SYMBOL_LEG_RRF_K = 180
-
-# The filename leg is a name match too, weighted like the symbol leg. A name the
-# question spells in full (``full_cover``, see ``_prose_symbols``) is fused at the
-# page legs' k.
-_FILENAME_LEG_RRF_K = _SYMBOL_LEG_RRF_K
-
-
-# ---------------------------------------------------------------------------
-# The question's embedding, computed once per call
-# ---------------------------------------------------------------------------
-
-# Answering one question used to embed its text up to three times: once for the
-# main fetch, once for the concept lookup on a subsystem-shaped question, once
-# for the neighbourhood re-rank on a flow-shaped one. Each is a network
-# round-trip to the embedding provider, billed, for a vector that cannot differ
-# between them.
-#
-# Keyed by ``(id(store), question)`` and holding the store in the value, so an
-# id can never be recycled onto a different store while its entries are live.
-# Capacity is a handful of entries rather than one: concurrent calls with
-# different questions would otherwise evict each other and re-embed. An entry is
-# a pure function of (embedder, text), so a hit across calls is as correct as a
-# hit within one.
-_QUESTION_VECTOR_CACHE_MAX = 4
-_QUESTION_VECTORS: OrderedDict[tuple[int, str], tuple[Any, list[float]]] = OrderedDict()
-
-
-async def question_vector(ctx: Any, question: str) -> list[float] | None:
- """The question's embedding, computed at most once per store and question.
-
- Returns None when there is no store, the backend holds no embedder of its
- own, or the embed failed — callers then fall back to the store's own
- ``search(text)``, which embeds inline. That fallback is a cost regression,
- never a correctness one, so it warns rather than raising.
- """
- store = getattr(ctx, "vector_store", None)
- if store is None or not question:
- return None
- if not store_has_semantic_vectors(store):
- # Nothing will consume it, so do not pay to compute it.
- return None
-
- key = (id(store), question)
- cached = _QUESTION_VECTORS.get(key)
- if cached is not None:
- _QUESTION_VECTORS.move_to_end(key)
- return cached[1]
-
- timeout_s = embed_timeout_s()
- try:
- vectors = await asyncio.wait_for(store.embed_texts([question]), timeout=timeout_s)
- except TimeoutError:
- # The A18 case, and the one worth naming separately: the embedder is
- # configured, reachable and healthy, and simply did not answer inside
- # the budget. Nothing downstream fails, so without this the answer is
- # lexical-only and indistinguishable from one that was not.
- _record_leg("embed", "timeout")
- _log.warning(
- "get_answer could not embed the question within %.1fs; retrieval "
- "continues without a question vector. Raise it with %s=<seconds>.",
- timeout_s,
- _EMBED_TIMEOUT_ENV,
- )
- return None
- except Exception:
- _record_leg("embed", "error")
- _log.warning(
- "get_answer could not embed the question up front; each retrieval "
- "stage will embed it again",
- exc_info=True,
- )
- return None
- if not vectors:
- # Backend without an embedder of its own. Not an error — but it means
- # every stage pays for its own round-trip, so it is worth seeing.
- _log.debug("Vector store cannot embed directly; per-stage embedding stands")
- return None
-
- vector = [float(v) for v in vectors[0]]
- _QUESTION_VECTORS[key] = (store, vector)
- while len(_QUESTION_VECTORS) > _QUESTION_VECTOR_CACHE_MAX:
- _QUESTION_VECTORS.popitem(last=False)
- return vector
-
-
-async def vector_search(
- store: Any, question: str, limit: int, *, vector: list[float] | None
-) -> list[Any]:
- """Nearest pages to *question*, reusing *vector* when the backend allows it.
-
- Every backend that can search by raw vector is spared a second embedding of
- text it has already embedded; one that cannot returns None from
- ``search_by_vector`` and is asked to search the text as before.
-
- Returns nothing on a keyless store. Guarded here rather than only at the
- callers because this is the shared floor under every vector read in the
- answer pipeline, and the text fallback on the last line would otherwise
- route a keyless store straight back into the embedder this is avoiding.
- """
- if not store_has_semantic_vectors(store):
- return []
- if vector is not None:
- by_vector = await store.search_by_vector(vector, limit=limit)
- if by_vector is not None:
- return by_vector
- return await store.search(question, limit=limit)
-
-
-# ---------------------------------------------------------------------------
-# Stage 1: Hybrid retrieval (FTS + vector → RRF merge)
-# ---------------------------------------------------------------------------
-
-
-async def hybrid_retrieve(question: str, ctx: Any) -> list[dict]:
- """Run FTS, vector, symbol and filename retrieval in parallel and merge via RRF.
-
- Returns a list of dicts shaped ``{page_id, title, score, snippet,
- page_type, _sources: set[str]}``. ``_sources`` names which retrievers
- found the hit — useful for ranking signal ("hit by both modes" is a
- stronger ground-truth signal than "hit by one"). Score is the RRF-fused
- score; downstream stages may multiply it further.
-
- Both retrievers are best-effort with timeouts so one slow path can never
- block the call. An empty result from one mode just means the other mode
- fully drives ranking, which matches the pre-hybrid behaviour.
-
- The symbol and filename legs also name indexed files that have no page, as
- ``page_type: "file"`` hits, the same rows ``search_codebase`` serves.
- """
- # Reset before the legs run so the record describes this question and not
- # a previous one that happened to share the task context.
- begin_leg_record()
- fts_task = _safe_fts_search(ctx, question)
- vec_task = _safe_vector_search(ctx, question)
- sym_task = _safe_symbol_search(ctx, question, pageless=True)
- name_task = _safe_filename_search(ctx, question)
- fts_results, vec_results, sym_results, name_results = await asyncio.gather(
- fts_task, vec_task, sym_task, name_task
- )
-
- # RRF merge. Each hit's contribution from a source is 1/(rank + k);
- # hits appearing in both sources sum their contributions naturally.
- # Per-source rank is preserved alongside the fused score: RRF *compresses*
- # scores (rank-0-in-both barely outscores rank-1-in-both), so the summed
- # number loses the "both retrievers independently ranked this #1" signal.
- # Downstream confidence uses these ranks to recover retriever *agreement*
- # as a dominance signal the numeric ratio can't see.
- fused: dict[str, dict] = {}
- for rank, h in enumerate(fts_results):
- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)
- entry["_sources"].add("fts")
- entry["_fts_rank"] = rank
- for rank, h in enumerate(vec_results):
- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)
- entry["_sources"].add("vector")
- entry["_vec_rank"] = rank
- for rank, h in leg_ranks(sym_results):
- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
- entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _SYMBOL_LEG_RRF_K)
- entry["_sources"].add("symbol")
- entry["_sym_rank"] = rank
- entry["_symbol_names"] = h.symbol_names
- # Adds files the other legs missed and never reorders a page they found.
- # Search fuses the same rows into its own entry shape, crediting a symbol
- # page of the file; here every leg is keyed by page id, as above.
- for rank, (h, full) in leg_ranks(name_results):
- if h.page_id in fused and h.page_type != PAGELESS_FILE:
- continue
- entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
- entry["score"] = entry.get("score", 0.0) + 1.0 / (
- rank + (_RRF_K if full else _FILENAME_LEG_RRF_K)
- )
- entry["_sources"].add("filename")
-
- # Scale to BM25-range so downstream confidence/dominance gates (tuned
- # against the prior single-mode BM25 retrieval) keep behaving sanely.
- # Ordering is unchanged — multiplying by a positive constant is a
- # no-op for ranking.
- for entry in fused.values():
- entry["score"] = entry["score"] * _RRF_SCORE_SCALE
-
- merged = list(fused.values())
- merged.sort(key=lambda h: h["score"], reverse=True)
- return merged
-
-
-def stamp_hybrid_rank(hits: list[dict]) -> None:
- """Record each hit's 0-based file rank in the fused order, before any rerank.
-
- This is the order ``search_codebase`` serves; confidence reads it to tell
- when the reranked lead left it. Symbol pages share their file's rank. Files
- are ranked among pages: a file with no page shares the rank of the page
- below it, so it never pushes a page down.
- """
- files: dict[str, int] = {}
- for h in hits:
- path = (h.get("target_path") or "").split("::", 1)[0]
- if h.get("page_type") == PAGELESS_FILE:
- h["_hybrid_rank"] = len(files)
- else:
- h["_hybrid_rank"] = files.setdefault(path, len(files))
-
-
-async def _safe_fts_search(ctx: Any, question: str) -> list[Any]:
- """FTS search wrapped in timeout + suppression. Returns [] on any failure."""
- if ctx.fts is None:
- _record_leg("fts", "absent")
- return []
- try:
- results = await asyncio.wait_for(
- ctx.fts.search(question, limit=_RETRIEVAL_FETCH_LIMIT), timeout=5.0
- )
- except TimeoutError:
- _record_leg("fts", "timeout")
- return []
- except Exception:
- _record_leg("fts", "error")
- return []
- _record_leg("fts", "ok")
- return results
-
-
-def _pages_only(results: list[Any], limit: int) -> list[Any]:
- """Best-first *results* with decision vectors removed, capped at *limit*.
-
- Decision records live in the page store under their own page-id namespace so
- dedup can match a paraphrase and ``search_codebase`` can surface a decision
- directly. Neither is true of answering: a decision row has no ``wiki_pages``
- row, so hydration leaves it pathless — and a pathless hit skips the tombstone
- check and the scope filter, can only be reordered by noise demotion rather
- than dropped, and cannot be cited by the answer it helped write.
-
- A why-shaped question still gets its decisions. They are injected from a
- path-overlap query over ``decision_records``, which does not involve this
- store at all.
- """
- kept = [r for r in results if not str(getattr(r, "page_id", "")).startswith(_DECISION_PREFIX)]
- dropped = len(results) - len(kept)
- if dropped:
- _log.debug(
- "get_answer page retrieval dropped %d decision vector(s) from a window of %d",
- dropped,
- len(results),
- )
- if len(kept) < limit and dropped:
- # The over-fetch margin was not enough. Reported because the visible
- # symptom is a short candidate set, which otherwise reads as a corpus
- # too small or a query too narrow.
- _log.warning(
- "get_answer page retrieval kept only %d of a requested %d candidates: %d "
- "decision vector(s) filled the window. Decisions may be over-represented "
- "in the vector store.",
- len(kept),
- limit,
- dropped,
- )
- return kept[:limit]
-
-
-async def _safe_vector_search(ctx: Any, question: str) -> list[Any]:
- """Vector search wrapped in timeout + suppression. Returns [] on any failure.
-
- Also waits for vector-store readiness when the lifespan event is set —
- skipping the wait would race a background-loading store on cold start.
-
- Returns nothing on a keyless index, before the readiness wait and before the
- question is embedded: there is no vector worth computing when there is
- nothing discriminative to compare it against. See
- ``store_has_semantic_vectors``. This leg is the only entry to vector
- retrieval here, so guarding it covers every caller.
- """
- if ctx.vector_store is None:
- _record_leg("vector", "absent")
- return []
- if not store_has_semantic_vectors(ctx.vector_store):
- _record_leg("vector", "keyless")
- return []
- ready = getattr(ctx, "vector_store_ready", None)
- if ready is not None:
- with contextlib.suppress(TimeoutError):
- await asyncio.wait_for(ready.wait(), timeout=30.0)
- # Embedded here, before the store is asked anything: this is the first stage
- # to need the vector, and every later stage reads the same one back.
- vector = await question_vector(ctx, question)
- try:
- results = await asyncio.wait_for(
- vector_search(
- ctx.vector_store,
- question,
- _RETRIEVAL_FETCH_LIMIT + _DECISION_OVERFETCH,
- vector=vector,
- ),
- timeout=vector_search_timeout_s(),
- )
- except TimeoutError:
- _record_leg("vector", "timeout")
- _log.warning(
- "Vector search exceeded its %gs budget; answering without semantic hits. "
- "Raise it with %s=<seconds>.",
- vector_search_timeout_s(),
- _VECTOR_TIMEOUT_ENV,
- )
- return []
- except Exception:
- _record_leg("vector", "error")
- return []
- _record_leg("vector", "ok")
- return _pages_only(results, _RETRIEVAL_FETCH_LIMIT)
-
-
-class _SymbolLegResult:
- """A file page reached through the symbol index, in retriever result shape.
-
- The RRF merge is keyed on page ids and reads four attributes off whatever
- the retrievers hand it, so the symbol leg presents its file pages the same
- way FTS and the vector store present theirs.
- """
-
- __slots__ = ("page_id", "page_type", "snippet", "symbol_names", "title")
-
- def __init__(
- self, page_id: str, title: str, snippet: str, page_type: str, symbol_names: list[str]
- ) -> None:
- self.page_id = page_id
- self.title = title
- self.snippet = snippet
- self.page_type = page_type
- self.symbol_names = symbol_names
-
-
-async def _safe_symbol_search(
- ctx: Any, question: str, *, pageless: bool = False
-) -> list[_SymbolLegResult]:
- """File pages whose symbols the question's words name. [] on any failure.
-
- Ungated: it runs on every question, not only on ones that happen to carry
- an identifier-shaped token. Which indexes get read is not something the
- grammar of the sentence should decide. "How does an incremental update
- persist symbols" and ``_persist_symbols`` are after the same file, and
- before this leg only the second one reached the symbol index.
-
- Best-effort with a timeout, like the other two legs: a slow or missing
- symbol index degrades ``get_answer`` to its previous behaviour rather than
- failing the call. ``pageless`` keeps files that have no page, as pageless
- rows, for a caller that can serve them.
- """
- try:
- pages = await asyncio.wait_for(
- symbol_backed_pages(
- ctx,
- question,
- max_files=_SYMBOL_LEG_MAX_PAGES,
- symbol_limit=_SYMBOL_LEG_FETCH_LIMIT,
- pageless=pageless,
- ),
- timeout=5.0,
- )
- except TimeoutError:
- _record_leg("symbol", "timeout")
- _log.debug("get_answer symbol leg timed out; page retrieval stands")
- return []
- except Exception:
- _record_leg("symbol", "error")
- _log.debug("get_answer symbol leg failed; page retrieval stands", exc_info=True)
- return []
- _record_leg("symbol", "ok")
- return [
- _SymbolLegResult(
- p["page_id"],
- p["title"],
- (p.get("summary") or "")[:200],
- p["page_type"],
- p.get("symbol_names") or [],
- )
- for p in pages
- ]
-
-
-async def _safe_filename_search(ctx: Any, question: str) -> list[tuple[_SymbolLegResult, bool]]:
- """Files whose file name the question spells, each with whether it spells all
- of the name. [] on any failure, like the other legs."""
- try:
- pages = await asyncio.wait_for(
- filename_backed_pages(ctx, question, max_files=_SYMBOL_LEG_MAX_PAGES), timeout=5.0
- )
- except TimeoutError:
- _record_leg("filename", "timeout")
- return []
- except Exception:
- _record_leg("filename", "error")
- _log.debug("filename leg failed; page retrieval stands", exc_info=True)
- return []
- _record_leg("filename", "ok")
- return [
- (
- _SymbolLegResult(p["page_id"], p["title"], p["summary"][:200], p["page_type"], []),
- p["full_cover"],
- )
- for p in pages
- ]
-
-
-def leg_ranks(results: list[Any]) -> list[tuple[int, Any]]:
- """Each result of a leg with its rank among that leg's pages.
-
- A file with no page takes the rank of the page below it, so it never costs
- a page its place in the leg: the pages fuse exactly as they would without it.
- Items may be results or ``(result, extra)`` pairs.
- """
- ranked: list[tuple[int, Any]] = []
- pages = 0
- for item in results:
- result = item[0] if isinstance(item, tuple) else item
- ranked.append((pages, item))
- if result.page_type != PAGELESS_FILE:
- pages += 1
- return ranked
-
-
-def _hit_dict_from_result(result: Any) -> dict:
- """Convert a retriever result object to the pipeline's dict shape."""
- return {
- "page_id": result.page_id,
- "title": getattr(result, "title", ""),
- "snippet": getattr(result, "snippet", ""),
- "page_type": getattr(result, "page_type", ""),
- "score": 0.0,
- "_sources": set(),
- }
-
-
-# ---------------------------------------------------------------------------
-# Noise demotion (decision records + test file pages)
-# ---------------------------------------------------------------------------
-
-# Retrieval noise that should not occupy get_answer's top-5 on a plain question.
-# Mirrors search_codebase's demotion (tool_search._sort_demoting_noise) on the
-# answer pipeline's hit shape. Both now read the same test-path rules, so the
-# two tools can no longer demote different sets of files.
-_TEST_QUERY_RE = re.compile(
- r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",
- re.IGNORECASE,
-)
-
-
-def demote_noise_hits(hits: list[dict], question: str, *, is_why: bool) -> list[dict]:
- """Stable-partition retrieval noise below real pages before the top-5 cap.
-
- get_answer applies no demotion of its own — decision records (short dense
- titles) and test file pages win RRF against the implementation a plain
- question is about and take top-5 slots that then feed synthesis. Decision
- records demote unless the question is why-shaped (decisions are the answer
- then, folded into the prelude); test file pages demote unless the question is
- explicitly about tests. Stable: real hits keep their order, and noise keeps
- its relative order at the tail (never dropped — an agent may still want it).
-
- Tests demote, test support does not: "where are the shared fixtures" is a
- plain question whose answer is a ``conftest.py``, and demoting it would push
- the answer out of the window that feeds synthesis.
- """
- if not hits:
- return hits
- test_focused = bool(_TEST_QUERY_RE.search(question))
-
- def _is_noise(h: dict) -> bool:
- pt = h.get("page_type")
- return (pt == "decision_record" and not is_why) or (
- pt in FILE_ROW_TYPES and not test_focused and is_test_path(h.get("target_path") or "")
- )
-
- real = [h for h in hits if not _is_noise(h)]
- noise = [h for h in hits if _is_noise(h)]
- return real + noise
-
-
-# ---------------------------------------------------------------------------
-# Stage 2: Hydrate hits with Page metadata (target_path, summary)
-# ---------------------------------------------------------------------------
-
-
-async def hydrate_hits(hits: list[dict], ctx: Any, *, scope: str | None = None) -> list[dict]:
- """Attach target_path, summary, and page_type from the Page table.
-
- Mutates each hit in place. Applies the ``scope`` filter (path prefix) at
- this stage rather than during retrieval — retrievers don't know about
- paths, and post-filtering keeps the merge logic source-agnostic.
- """
- if not hits:
- return hits
- page_ids = [h["page_id"] for h in hits]
- async with get_session(ctx.session_factory) as session:
- res = await session.execute(
- select(
- Page.id,
- Page.target_path,
- Page.summary,
- Page.page_type,
- Page.freshness_status,
- ).where(Page.id.in_(page_ids))
- )
- meta_by_id = {
- row[0]: {
- "target_path": row[1] or "",
- "summary": row[2] or "",
- "page_type": row[3] or "",
- "freshness": row[4] or "",
- }
- for row in res.all()
- }
-
- out: list[dict] = []
- pageless = 0
- for h in hits:
- meta = meta_by_id.get(h["page_id"])
- if meta is None and h.get("page_type") == PAGELESS_FILE:
- # An indexed file with no page: its path is in its id, and it has
- # no summary to give, so it serves its matched code instead.
- meta = {"target_path": pageless_path(h["page_id"]), "page_type": PAGELESS_FILE}
- if meta is None:
- # A retrieved id with no page behind it: a decision vector, or a
- # vector left over from a page the stores have since disagreed
- # about. Either way there is nothing to cite and nothing to read,
- # and keeping it costs a served slot — while every later gate that
- # would have caught it (tombstone, scope) is keyed on a path it
- # does not have.
- pageless += 1
- continue
- # Tombstoned pages document deleted/renamed files — serving them as
- # answer material would cite code that no longer exists.
- if meta.get("freshness") == "tombstone":
- continue
- target_path = meta.get("target_path", "")
- if scope and target_path and not target_path.startswith(scope):
- continue
- h["target_path"] = target_path
- h["summary"] = meta.get("summary", "")
- # Prefer the Page table's page_type when present — it's the source
- # of truth; retrievers sometimes carry stale or empty types.
- h["page_type"] = meta.get("page_type") or h.get("page_type", "")
- out.append(h)
- if pageless:
- _log.warning(
- "get_answer dropped %d of %d retrieved hit(s) with no page row behind them; "
- "an id in a retriever that the page table does not know is either a "
- "non-page vector or a three-store disagreement (repowise doctor --repair)",
- pageless,
- len(hits),
- )
- return out
-
-
-# ---------------------------------------------------------------------------
-# Stage 3: PageRank bias
-# ---------------------------------------------------------------------------
-
-
-async def apply_pagerank_bias(hits: list[dict], ctx: Any) -> None:
- """Multiply each hit's score by a damped PageRank factor (in place).
-
- Looks up the GraphNode row for each hit's target_path and pulls its
- PageRank. We normalise within the candidate set rather than against the
- whole graph: a candidate with the highest PageRank among its peers gets
- the full bias, the lowest gets none. This avoids the failure mode where
- an absolute scale would over-reward famous-but-irrelevant files.
- """
- if not hits:
- return
- paths = [h.get("target_path") for h in hits if h.get("target_path")]
- if not paths:
- return
- async with get_session(ctx.session_factory) as session:
- # Look up GraphNodes by node_id — file nodes are keyed by their path.
- res = await session.execute(
- select(GraphNode.node_id, GraphNode.pagerank).where(
- GraphNode.node_id.in_(paths),
- GraphNode.node_type == "file",
- )
- )
- pr_by_path = {row[0]: float(row[1] or 0.0) for row in res.all()}
-
- if not pr_by_path:
- return
- # Normalised over the pages, so a file with no page cannot change their bias.
- paged = {h.get("target_path") for h in hits if h.get("page_type") != PAGELESS_FILE}
- max_pr = max((pr for path, pr in pr_by_path.items() if path in paged), default=0.0)
- max_pr = max_pr or max(pr_by_path.values(), default=0.0)
- if max_pr <= 0:
- return
-
- for h in hits:
- pr = pr_by_path.get(h.get("target_path"), 0.0)
- # Normalised in [0, 1] then scaled to a multiplicative bias in
- # [1.0, 1 + _PAGERANK_BIAS_MAX].
- bias = 1.0 + _PAGERANK_BIAS_MAX * min(pr / max_pr, 1.0)
- h["_pagerank"] = pr
- h["_pagerank_bias"] = round(bias, 3)
- h["score"] = h.get("score", 0.0) * bias
-
- hits.sort(key=lambda h: h["score"], reverse=True)
-
-
-# ---------------------------------------------------------------------------
-# Stage 4: Graph expansion (1-hop neighbors of top hits)
-# ---------------------------------------------------------------------------
-
-
-def _projected_pair(
- src: str | None, tgt: str | None, edge_type: str | None, confidence: float | None
-) -> tuple[str | None, str | None]:
- """An edge's two endpoints as file paths, or ``(None, None)`` to drop it.
-
- Only the symbol layer gets the guards; the file layer is already
- file-to-file and keeps today's behaviour. The asymmetry is deliberate — the
- guards refuse cross-extension pairs, and a ``co_changes`` edge between a
- module and its docs is a "read this too" expansion has always served.
- """
- src = src or ""
- tgt = tgt or ""
- if not is_symbol_node(src) and not is_symbol_node(tgt):
- return (src or None, tgt or None)
- src_file, tgt_file = node_to_file(src), node_to_file(tgt)
- if not keep_projected_edge(src_file, tgt_file, edge_type, confidence):
- return (None, None)
- return (src_file, tgt_file)
-
-
-async def _projected_edges(session: Any, repo_id: str) -> list[tuple[str, str]]:
- """Every non-containment edge for *repo_id*, as a pair of file paths.
-
- One scan, projected in Python. A prefix match on the symbol layer cannot use
- the node-id index (SQLite disables the LIKE optimisation under an ESCAPE
- clause), so per-candidate queries were a scan each; this is one.
-
- Containment is excluded because both its endpoints describe the same file,
- so a projected containment edge is always a self-loop.
- """
- res = await session.execute(
- select(
- GraphEdge.source_node_id,
- GraphEdge.target_node_id,
- GraphEdge.edge_type,
- GraphEdge.confidence,
- ).where(
- GraphEdge.repository_id == repo_id,
- GraphEdge.edge_type.notin_(CONTAINMENT_EDGE_TYPES),
- )
- )
- pairs: list[tuple[str, str]] = []
- for src, tgt, etype, conf in res.all():
- src_file, tgt_file = _projected_pair(src, tgt, etype, conf)
- if src_file and tgt_file:
- pairs.append((src_file, tgt_file))
- return pairs
-
-
-def _hub_degree_cutoff(degree: dict[str, int]) -> int:
- """Degree at which a candidate counts as a hub, for this candidate set.
-
- ``max(floor, p99)``. The floor keeps a small or sparsely-linked repo from
- excluding ordinary files, and the percentile keeps a densely-linked one
- from excluding nothing.
- """
- if not degree:
- return 1 << 30
- values = sorted(degree.values())
- idx = min(len(values) - 1, int(len(values) * _HUB_DEGREE_PERCENTILE))
- return max(_HUB_DEGREE_FLOOR, values[idx])
-
-
-async def expand_via_graph(hits: list[dict], ctx: Any, repo_id: str) -> list[dict]:
- """Add up to ``_GRAPH_EXPAND_MAX_NEW`` graph-neighbor files to ``hits``.
-
- Rescues near-misses where the top retrieved file is in the right
- neighborhood but isn't the actual answer (a consumer instead of the
- orchestrator, a wrapper instead of the implementation). Expansion walks
- one hop through GraphEdges in both directions from the top-N candidates,
- then folds in any neighbor that:
- * has a wiki page (otherwise the LLM has nothing to read)
- * is not already in the candidate set
-
- Expanded hits carry an ``_expanded_from`` marker and a damped score so
- the gate / confidence calibration knows they're indirect.
- """
- if not hits:
- return hits
- # Seeded from pages only, so a file with no page cannot change what pages add.
- seeds = [h for h in hits if h.get("page_type") != PAGELESS_FILE][:_GRAPH_EXPAND_TOP_N]
- seed_paths = [h.get("target_path") for h in seeds if h.get("target_path")]
- if not seed_paths:
- return hits
- existing = {h.get("target_path") for h in hits}
-
- async with get_session(ctx.session_factory) as session:
- # Not import edges only, despite the naming this code used to carry:
- # ``co_changes`` neighbours stay in on purpose, because "this file moves
- # with yours" is a genuine read-this-too signal, and expansion surfaces
- # what it adds neutrally as ``[graph-expanded]`` rather than as an import
- # claim.
- #
- # The scan reaches the symbol layer as well as the file layer: ``calls``
- # and its siblings join ``path::Name`` nodes, so an equality test against
- # a seed path matched none of them and the call graph was invisible here.
- seed_set = set(seed_paths)
- pairs = await per_index(
- session, repo_id, "projected_edges", lambda: _projected_edges(session, repo_id)
- )
-
- neighbors: set[str] = set()
- degree: dict[str, int] = {}
- for src_file, tgt_file in pairs:
- degree[src_file] = degree.get(src_file, 0) + 1
- degree[tgt_file] = degree.get(tgt_file, 0) + 1
- for mine, theirs in ((src_file, tgt_file), (tgt_file, src_file)):
- if mine in seed_set and theirs not in existing:
- neighbors.add(theirs)
-
- if not neighbors:
- return hits
-
- # Only fold in neighbors that have a wiki page — otherwise the LLM
- # context block can't carry a useful excerpt for them.
- page_res = await session.execute(
- select(Page.target_path, Page.summary, Page.page_type).where(
- Page.target_path.in_(sorted(neighbors)),
- Page.page_type == "file_page",
- )
- )
- page_rows = list(page_res.all())
-
- # Also load PageRank for the neighbors so we can rank them.
- pr_res = await session.execute(
- select(GraphNode.node_id, GraphNode.pagerank).where(
- GraphNode.node_id.in_(sorted(neighbors)),
- GraphNode.node_type == "file",
- )
- )
- pr_by_path = {row[0]: float(row[1] or 0.0) for row in pr_res.all()}
-
- if not page_rows:
- return hits
-
- # Drop hubs before ranking. Ranking by PageRank alone actively prefers
- # them, which is the wrong instinct here: expansion is trying to name the
- # specific file the question is about, and the most-imported file in the
- # repo is the least specific candidate available.
- # Scoped to the candidate set, not the repo: the cutoff answers "is this
- # neighbour a hub among the ones on offer".
- cutoff = _hub_degree_cutoff({row[0]: degree.get(row[0], 0) for row in page_rows})
- non_hub = [row for row in page_rows if degree.get(row[0], 0) <= cutoff]
- if non_hub:
- page_rows = non_hub
-
- # Damp parent score by _GRAPH_EXPAND_DAMPING for child candidates; pick
- # the strongest parent each child connects to (taking the max parent
- # score is conservative — favors well-connected neighbors).
- strongest_parent = max(
- seeds, key=lambda hit: hit.get("score", 0.0), default={}
- )
- parent_score = strongest_parent.get("score", 0.0)
- confidence_factor = strongest_parent.get("_confidence_score_factor")
- candidates: list[dict] = []
- for path, summary, page_type in page_rows:
- candidate = {
- "page_id": f"file_page:{path}",
- "target_path": path,
- "title": f"File: {path}",
- "summary": summary or "",
- "snippet": (summary or "")[:200],
- "page_type": page_type or "file_page",
- "score": parent_score * _GRAPH_EXPAND_DAMPING,
- "_sources": {"graph_expand"},
- "_expanded_from": "graph",
- "_pagerank": pr_by_path.get(path, 0.0),
- }
- if confidence_factor is not None:
- candidate["_confidence_score_factor"] = confidence_factor
- candidates.append(candidate)
-
- # Rank candidates by PageRank within the expansion set so we pick the
- # most central neighbor first when we have multiple plausible ones.
- # Path breaks ties, so which neighbour is picked cannot move with the
- # database's row order between runs on identical data.
- candidates.sort(key=lambda c: (-c.get("_pagerank", 0.0), c["target_path"]))
- additions = candidates[:_GRAPH_EXPAND_MAX_NEW]
- if not additions:
- return hits
-
- combined = hits + additions
- combined.sort(key=lambda h: h["score"], reverse=True)
- return combined
-
-
-# ---------------------------------------------------------------------------
-# Stage: parent-concept surfacing (subsystem-shaped questions)
-# ---------------------------------------------------------------------------
-
-# A subsystem-shaped question asks about a part of the system as a whole:
-# "overview of X", "main parts of X", "what subsystem does Y belong to",
-# "where would I add a Z". For these the best answer is the concept page that
-# documents the whole subsystem, not one of its member files or a more specific
-# child concept page. Retrieval ranks the specific children above the parent (a
-# child's embedding matches the query's noun more tightly than the broader
-# parent), so the parent concept/rollup page never surfaces even though it
-# exists. This gate is purely on the query's natural-language shape — no
-# repo-specific vocabulary — so it generalises across codebases.
-# Kept deliberately high-precision: bare fragments like "parts of", "belongs to",
-# "structure of", or "which module" appear constantly inside ordinary how/where
-# questions ("what parts of the code touch this", "which module imports config"),
-# so each phrase is anchored to an unambiguous subsystem-overview intent. Better
-# to miss an oddly-worded subsystem question (it just keeps today's ranking) than
-# to reorder an implementation question's file hits.
-_SUBSYSTEM_QUERY_RE = re.compile(
- r"\boverview of\b|\bgive me an overview\b|\bhigh[- ]level\b|"
- r"\bmain parts of\b|\bwhat (?:are|is) the (?:main )?(?:parts|pieces|components) of\b|"
- r"\barchitecture of\b|\bwalk me through\b|"
- r"\bhow (?:is|are|does) .+ (?:organi[sz]ed|structured|laid out|put together)\b|"
- r"\bwhat subsystem\b|\bwhich subsystem\b|"
- r"\bwhere would i (?:add|put)\b|\bwhere do i (?:add|put)\b",
- re.IGNORECASE,
-)
-
-# How many of the strongest real (non-noise) hits to cluster when looking for
-# their shared subsystem. Decision records and pages without a path are skipped
-# so a query whose top slots are crowded by decision noise still clusters on its
-# real member files.
-_PARENT_EXPAND_TOP_N = 8
-# A directory is a tight structural cluster when it is the immediate parent of at
-# least this many surfaced hits. A lone surfaced file is not enough structural
-# signal on its own; that case is left to the semantic concept lookup.
-_PARENT_MIN_SHARE = 2
-# The injected parent leads the surface (it is the answer) but only just: a
-# small multiplier over the current top score keeps it first without
-# manufacturing a dominant retrieval that would inflate confidence to "high".
-_PARENT_EXPAND_BOOST = 1.02
-# How deep to look in a concept-restricted vector search for the subsystem page
-# a query is semantically about. The window is wide because concept pages are a
-# small minority of the corpus, so the right one can sit below many file/symbol
-# hits before this filter drops those away.
-_CONCEPT_FETCH_LIMIT = 60
-
-
-def is_subsystem_query(question: str) -> bool:
- """True when the question asks about a subsystem/module as a whole, so the
- concept page for that subsystem should lead rather than its member files."""
- return bool(_SUBSYSTEM_QUERY_RE.search(question or ""))
-
-
-def _common_ancestor(paths: set[str]) -> str:
- """Longest shared directory prefix of the given paths, segment-wise."""
- split = [p.split("/") for p in paths]
- common: list[str] = []
- for segs in zip(*split, strict=False):
- if len(set(segs)) == 1:
- common.append(segs[0])
- else:
- break
- return "/".join(common)
-
-
-async def _semantic_concept_paths(question: str, ctx: Any) -> list[str]:
- """Target paths of the concept/layer pages a concept-restricted vector search
- ranks highest for the question, best-first.
-
- This is the "guaranteed concept-page candidate": when a subsystem query's
- file hits land in the wrong neighborhood (a UI consumer of the subsystem
- rather than the subsystem itself), the subsystem's own concept page still
- matches the query semantically and surfaces here even though it never
- entered the file-dominated main fetch. Best-effort: returns [] on any error
- so the caller degrades to the structural path.
- """
- vs = getattr(ctx, "vector_store", None)
- if vs is None:
- return []
- try:
- results = await asyncio.wait_for(
- vector_search(
- vs, question, _CONCEPT_FETCH_LIMIT, vector=await question_vector(ctx, question)
- ),
- timeout=vector_search_timeout_s(),
- )
- except Exception:
- return []
- out: list[str] = []
- for r in results:
- if getattr(r, "page_type", "") in ("module_page", "layer_page"):
- tp = getattr(r, "target_path", "") or ""
- if not tp:
- pid = getattr(r, "page_id", "") or ""
- tp = pid.split(":", 1)[1] if ":" in pid else pid
- if tp and tp not in out:
- out.append(tp)
- return out
-
-
-async def expand_via_parent_page(hits: list[dict], question: str, ctx: Any) -> list[dict]:
- """Lead a subsystem-shaped question with the concept page for its subsystem.
-
- Two complementary signals pick that page, neither tuned to any repository:
-
- * Structural — when the surfaced file hits cluster under an ancestor
- directory that has a concept/rollup page, that ancestor IS the subsystem.
- Ancestors are found by walking each hit's target_path up the tree (the
- same relationship the generator uses to mint rollups). The tightest
- (deepest) ancestor covering >=2 hits wins, never a catch-all near the root.
- * Semantic — when the file hits land in the wrong neighborhood (a consumer
- of the subsystem, not the subsystem), the subsystem's own concept page
- still matches the query and is recovered by a concept-restricted vector
- search. Used only when no structural cluster is found, so a strong file
- cluster is never overridden by a semantic guess.
-
- The chosen page is promoted in place if already retrieved (its children
- out-embed it, so the cap drops it) or injected as the leading hit otherwise.
- A no-op on every non-subsystem question, so file/implementation queries keep
- today's ranking untouched.
- """
- if not hits or not is_subsystem_query(question):
- return hits
- # Cluster on the strongest real hits: skip decision records and any hit
- # without a path so decision noise crowding the top slots can't starve the
- # clustering of the member files that reveal the subsystem.
- top = [h for h in hits if h.get("target_path") and h.get("page_type") != "decision_record"][
- :_PARENT_EXPAND_TOP_N
- ]
- if not top:
- return hits
-
- # For each surfaced hit, count its immediate parent directory and credit
- # every strict ancestor with covering it. A dir that is the immediate parent
- # of two or more surfaced hits is a TIGHT cluster: the query's own member
- # files sit directly in it. Coverage by a distant ancestor (a broad root that
- # merely contains scattered hits) is deliberately not enough — that is what
- # separates a real subsystem from the repository root.
- imm_count: dict[str, int] = {}
- covers: dict[str, set[str]] = {}
- for h in top:
- tp = h["target_path"].rstrip("/")
- parent = tp.rsplit("/", 1)[0] if "/" in tp else ""
- if parent:
- imm_count[parent] = imm_count.get(parent, 0) + 1
- anc = parent
- while anc:
- covers.setdefault(anc, set()).add(tp)
- anc = anc.rsplit("/", 1)[0] if "/" in anc else ""
-
- tight_clusters = {d for d, c in imm_count.items() if c >= _PARENT_MIN_SHARE}
- semantic_paths = await _semantic_concept_paths(question, ctx)
- candidates = set(covers) | set(imm_count) | set(semantic_paths)
- if not candidates:
- return hits
-
- async with get_session(ctx.session_factory) as session:
- rows = (
- await session.execute(
- select(Page.target_path, Page.title, Page.summary, Page.page_type).where(
- Page.target_path.in_(candidates),
- Page.page_type.in_(("module_page", "layer_page")),
- )
- )
- ).all()
- if not rows:
- return hits
- by_path = {r[0]: r for r in rows}
-
- # A tight structural cluster is the query's own files agreeing on a subsystem,
- # so it outranks the semantic guess. When several sibling dirs each cluster
- # (a subsystem split into subdirectories), roll up to their common ancestor
- # page so the answer is the subsystem, not one arbitrary half of it. With no
- # tight cluster the file hits landed in the wrong neighborhood, so the concept
- # page the query is semantically about wins instead.
- tight_pages = tight_clusters & by_path.keys()
- winner = None
- if tight_pages:
- rollup = _common_ancestor(tight_pages)
- if len(tight_pages) > 1 and rollup in by_path and rollup in covers:
- winner = by_path[rollup]
- else:
- best_tp = max(tight_pages, key=lambda d: (imm_count[d], d.count("/")))
- winner = by_path[best_tp]
- else:
- # No tight cluster: the semantically-closest concept page (best-first),
- # then any weak cover ancestor as a last resort.
- winner = next((by_path[tp] for tp in semantic_paths if tp in by_path), None)
- if winner is None:
- cov_pages = [a for a in covers if a in by_path]
- if cov_pages:
- winner = by_path[max(cov_pages, key=lambda a: (a.count("/"), len(covers[a])))]
- if winner is None:
- return hits
-
- best_tp, best_title, best_summary, best_pt = winner
- top_score = max((h.get("score", 0.0) for h in hits), default=0.0) or 1.0
- lead_score = top_score * _PARENT_EXPAND_BOOST
-
- # Already in the candidate set but ranked below its own children (they embed
- # tighter to the query noun than the broader parent), so the top-5 cap drops
- # it. Promote it to lead in place rather than adding a duplicate. The bump is
- # small (just past the top hit) so it never manufactures a dominant retrieval
- # that would read "high".
- for h in hits:
- if h.get("target_path") == best_tp:
- if h.get("score", 0.0) < lead_score:
- h["score"] = lead_score
- src = h.get("_sources")
- if isinstance(src, set):
- src.add("parent_promote")
- hits.sort(key=lambda x: x.get("score", 0.0), reverse=True)
- return hits
-
- # Not retrieved at all: inject it as the leading hit.
- parent_hit = {
- "page_id": f"{best_pt}:{best_tp}",
- "target_path": best_tp,
- "title": best_title or f"Overview: {best_tp}",
- "summary": best_summary or "",
- "snippet": (best_summary or "")[:200],
- "page_type": best_pt or "module_page",
- "score": lead_score,
- "_sources": {"parent_expand"},
- "_expanded_from": "parent",
- }
- combined = [parent_hit, *hits]
- combined.sort(key=lambda h: h["score"], reverse=True)
- return combined
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.py
new file mode 100644
index 000000000..db1b1bf5f
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/__init__.py
@@ -0,0 +1,420 @@
+"""Retrieval pipeline for ``get_answer``: question in, ranked wiki hits out.
+
+The LLM step, cache and response shape live in ``tool_answer``. Stages, each a
+function over hit dicts:
+
+1. ``hybrid_retrieve``: FTS, vector, symbol and filename legs in parallel,
+ fused by Reciprocal Rank Fusion. The page legs only see what a generated
+ page spells out; the symbol and filename legs recover private helpers and
+ local names.
+2. ``hydrate_hits`` (``hydrate``): attach path, summary and page type.
+3. ``apply_pagerank_bias`` (``hydrate``): central files win near-ties.
+4. ``expand_via_graph`` (``graph_expand``): add 1-hop neighbours of the top hits.
+5. ``expand_via_parent_page`` (``parent_page``): a subsystem question leads with
+ its concept page.
+
+``noise`` demotes decision and test pages; ``vectors`` and ``leg_record`` are
+shared by the legs. The legs and their merge live here, so a leg replaced on
+this module is the one ``hybrid_retrieve`` calls.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from typing import Any
+
+from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX
+from repowise.core.providers.embedding import store_has_semantic_vectors
+from repowise.server.mcp_server._answer_pipeline.graph_expand import (
+ _GRAPH_EXPAND_DAMPING,
+ _GRAPH_EXPAND_MAX_NEW,
+ _GRAPH_EXPAND_TOP_N,
+ _HUB_DEGREE_FLOOR,
+ _HUB_DEGREE_PERCENTILE,
+ _hub_degree_cutoff,
+ _projected_edges,
+ _projected_pair,
+ expand_via_graph,
+)
+from repowise.server.mcp_server._answer_pipeline.hydrate import (
+ _PAGERANK_BIAS_MAX,
+ apply_pagerank_bias,
+ hydrate_hits,
+)
+from repowise.server.mcp_server._answer_pipeline.leg_record import (
+ _LEG_RECORD,
+ _record_leg,
+ begin_leg_record,
+ degraded_legs,
+ retrieval_legs,
+)
+from repowise.server.mcp_server._answer_pipeline.noise import _TEST_QUERY_RE, demote_noise_hits
+from repowise.server.mcp_server._answer_pipeline.parent_page import (
+ _CONCEPT_FETCH_LIMIT,
+ _PARENT_EXPAND_BOOST,
+ _PARENT_EXPAND_TOP_N,
+ _PARENT_MIN_SHARE,
+ _SUBSYSTEM_QUERY_RE,
+ _common_ancestor,
+ _semantic_concept_paths,
+ expand_via_parent_page,
+ is_subsystem_query,
+)
+from repowise.server.mcp_server._answer_pipeline.vectors import (
+ _QUESTION_VECTOR_CACHE_MAX,
+ _QUESTION_VECTORS,
+ question_vector,
+ vector_search,
+)
+from repowise.server.mcp_server._helpers import _VECTOR_TIMEOUT_ENV, vector_search_timeout_s
+from repowise.server.mcp_server._page_paths import PAGELESS_FILE
+from repowise.server.mcp_server._prose_symbols import filename_backed_pages, symbol_backed_pages
+
+_log = logging.getLogger("repowise.mcp.answer")
+
+# Candidates each leg fetches before fusion.
+_RETRIEVAL_FETCH_LIMIT = 15
+
+# Decision vectors share the page store under this page-id prefix.
+_DECISION_PREFIX = DECISION_VECTOR_PREFIX
+
+# Extra rows the vector leg fetches so dropping decision rows still leaves a
+# full window; decision rows rank well on question-shaped text.
+_DECISION_OVERFETCH = 15
+
+# Standard RRF k: adjacent ranks score close, a one-leg rank-10 hit still counts.
+_RRF_K = 60
+
+# Scales fused scores into the BM25 range the downstream confidence gates were
+# tuned on (top RRF 1/60 + 1/60 ~ 0.033 -> ~6). Never changes the order.
+_RRF_SCORE_SCALE = 180.0
+
+# Symbol leg size: fewer than the page legs, it adds what they cannot see.
+_SYMBOL_LEG_FETCH_LIMIT = 20
+_SYMBOL_LEG_MAX_PAGES = 8
+
+# A larger k for name-match legs: they can lift or add a file, never outvote
+# two page legs that agree.
+_SYMBOL_LEG_RRF_K = 180
+
+# A name the question spells in full (``full_cover``) fuses at the page legs' k.
+_FILENAME_LEG_RRF_K = _SYMBOL_LEG_RRF_K
+
+
+async def hybrid_retrieve(question: str, ctx: Any) -> list[dict]:
+ """Run the four legs in parallel and fuse them by RRF, best first.
+
+ Hits are ``{page_id, title, score, snippet, page_type, _sources}`` plus each
+ leg's rank (``_fts_rank``, ``_vec_rank``, ``_sym_rank``): RRF compresses
+ scores, and confidence reads leg agreement from the ranks. A leg that fails
+ or times out contributes nothing. Pageless files come back as
+ ``page_type: "file"`` hits.
+ """
+ # Reset so the record describes this question, not one sharing the context.
+ begin_leg_record()
+ fts_task = _safe_fts_search(ctx, question)
+ vec_task = _safe_vector_search(ctx, question)
+ sym_task = _safe_symbol_search(ctx, question, pageless=True)
+ name_task = _safe_filename_search(ctx, question)
+ fts_results, vec_results, sym_results, name_results = await asyncio.gather(
+ fts_task, vec_task, sym_task, name_task
+ )
+
+ fused: dict[str, dict] = {}
+ for rank, h in enumerate(fts_results):
+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)
+ entry["_sources"].add("fts")
+ entry["_fts_rank"] = rank
+ for rank, h in enumerate(vec_results):
+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _RRF_K)
+ entry["_sources"].add("vector")
+ entry["_vec_rank"] = rank
+ for rank, h in leg_ranks(sym_results):
+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
+ entry["score"] = entry.get("score", 0.0) + 1.0 / (rank + _SYMBOL_LEG_RRF_K)
+ entry["_sources"].add("symbol")
+ entry["_sym_rank"] = rank
+ entry["_symbol_names"] = h.symbol_names
+ # Adds files the other legs missed; never reorders a page they found.
+ for rank, (h, full) in leg_ranks(name_results):
+ if h.page_id in fused and h.page_type != PAGELESS_FILE:
+ continue
+ entry = fused.setdefault(h.page_id, _hit_dict_from_result(h))
+ entry["score"] = entry.get("score", 0.0) + 1.0 / (
+ rank + (_RRF_K if full else _FILENAME_LEG_RRF_K)
+ )
+ entry["_sources"].add("filename")
+
+ for entry in fused.values():
+ entry["score"] = entry["score"] * _RRF_SCORE_SCALE
+
+ merged = list(fused.values())
+ merged.sort(key=lambda h: h["score"], reverse=True)
+ return merged
+
+
+def stamp_hybrid_rank(hits: list[dict]) -> None:
+ """Record each hit's 0-based file rank in the fused order, before any rerank.
+
+ Symbol pages share their file's rank; a pageless file shares the rank of the
+ page below it, so it never pushes a page down.
+ """
+ files: dict[str, int] = {}
+ for h in hits:
+ path = (h.get("target_path") or "").split("::", 1)[0]
+ if h.get("page_type") == PAGELESS_FILE:
+ h["_hybrid_rank"] = len(files)
+ else:
+ h["_hybrid_rank"] = files.setdefault(path, len(files))
+
+
+async def _safe_fts_search(ctx: Any, question: str) -> list[Any]:
+ """FTS leg; [] on timeout or failure."""
+ if ctx.fts is None:
+ _record_leg("fts", "absent")
+ return []
+ try:
+ results = await asyncio.wait_for(
+ ctx.fts.search(question, limit=_RETRIEVAL_FETCH_LIMIT), timeout=5.0
+ )
+ except TimeoutError:
+ _record_leg("fts", "timeout")
+ return []
+ except Exception:
+ _record_leg("fts", "error")
+ return []
+ _record_leg("fts", "ok")
+ return results
+
+
+def _pages_only(results: list[Any], limit: int) -> list[Any]:
+ """Best-first *results* without decision vectors, capped at *limit*.
+
+ A decision row has no page, so it could not be cited, scoped or tombstoned.
+ Why-shaped questions get decisions from ``decision_records`` instead.
+ """
+ kept = [r for r in results if not str(getattr(r, "page_id", "")).startswith(_DECISION_PREFIX)]
+ dropped = len(results) - len(kept)
+ if dropped:
+ _log.debug(
+ "get_answer page retrieval dropped %d decision vector(s) from a window of %d",
+ dropped,
+ len(results),
+ )
+ if len(kept) < limit and dropped:
+ # A short window would otherwise read as thin retrieval.
+ _log.warning(
+ "get_answer page retrieval kept only %d of a requested %d candidates: %d "
+ "decision vector(s) filled the window. Decisions may be over-represented "
+ "in the vector store.",
+ len(kept),
+ limit,
+ dropped,
+ )
+ return kept[:limit]
+
+
+async def _safe_vector_search(ctx: Any, question: str) -> list[Any]:
+ """Vector leg; [] on a keyless store, timeout or failure.
+
+ Waits for a store still loading at startup, and checks for a keyless store
+ before embedding anything.
+ """
+ if ctx.vector_store is None:
+ _record_leg("vector", "absent")
+ return []
+ if not store_has_semantic_vectors(ctx.vector_store):
+ _record_leg("vector", "keyless")
+ return []
+ ready = getattr(ctx, "vector_store_ready", None)
+ if ready is not None:
+ with contextlib.suppress(TimeoutError):
+ await asyncio.wait_for(ready.wait(), timeout=30.0)
+ vector = await question_vector(ctx, question)
+ try:
+ results = await asyncio.wait_for(
+ vector_search(
+ ctx.vector_store,
+ question,
+ _RETRIEVAL_FETCH_LIMIT + _DECISION_OVERFETCH,
+ vector=vector,
+ ),
+ timeout=vector_search_timeout_s(),
+ )
+ except TimeoutError:
+ _record_leg("vector", "timeout")
+ _log.warning(
+ "Vector search exceeded its %gs budget; answering without semantic hits. "
+ "Raise it with %s=<seconds>.",
+ vector_search_timeout_s(),
+ _VECTOR_TIMEOUT_ENV,
+ )
+ return []
+ except Exception:
+ _record_leg("vector", "error")
+ return []
+ _record_leg("vector", "ok")
+ return _pages_only(results, _RETRIEVAL_FETCH_LIMIT)
+
+
+class _SymbolLegResult:
+ """A file page from a name leg, in the shape FTS and vector results have."""
+
+ __slots__ = ("page_id", "page_type", "snippet", "symbol_names", "title")
+
+ def __init__(
+ self, page_id: str, title: str, snippet: str, page_type: str, symbol_names: list[str]
+ ) -> None:
+ self.page_id = page_id
+ self.title = title
+ self.snippet = snippet
+ self.page_type = page_type
+ self.symbol_names = symbol_names
+
+
+async def _safe_symbol_search(
+ ctx: Any, question: str, *, pageless: bool = False
+) -> list[_SymbolLegResult]:
+ """File pages whose symbols the question's words name; [] on any failure.
+
+ Runs on every question, not only identifier-shaped ones. ``pageless`` keeps
+ files with no page for a caller that can serve them.
+ """
+ try:
+ pages = await asyncio.wait_for(
+ symbol_backed_pages(
+ ctx,
+ question,
+ max_files=_SYMBOL_LEG_MAX_PAGES,
+ symbol_limit=_SYMBOL_LEG_FETCH_LIMIT,
+ pageless=pageless,
+ ),
+ timeout=5.0,
+ )
+ except TimeoutError:
+ _record_leg("symbol", "timeout")
+ _log.debug("get_answer symbol leg timed out; page retrieval stands")
+ return []
+ except Exception:
+ _record_leg("symbol", "error")
+ _log.debug("get_answer symbol leg failed; page retrieval stands", exc_info=True)
+ return []
+ _record_leg("symbol", "ok")
+ return [
+ _SymbolLegResult(
+ p["page_id"],
+ p["title"],
+ (p.get("summary") or "")[:200],
+ p["page_type"],
+ p.get("symbol_names") or [],
+ )
+ for p in pages
+ ]
+
+
+async def _safe_filename_search(ctx: Any, question: str) -> list[tuple[_SymbolLegResult, bool]]:
+ """Files the question names, each with whether it spells the whole name; [] on failure."""
+ try:
+ pages = await asyncio.wait_for(
+ filename_backed_pages(ctx, question, max_files=_SYMBOL_LEG_MAX_PAGES), timeout=5.0
+ )
+ except TimeoutError:
+ _record_leg("filename", "timeout")
+ return []
+ except Exception:
+ _record_leg("filename", "error")
+ _log.debug("filename leg failed; page retrieval stands", exc_info=True)
+ return []
+ _record_leg("filename", "ok")
+ return [
+ (
+ _SymbolLegResult(p["page_id"], p["title"], p["summary"][:200], p["page_type"], []),
+ p["full_cover"],
+ )
+ for p in pages
+ ]
+
+
+def leg_ranks(results: list[Any]) -> list[tuple[int, Any]]:
+ """Each leg result with its rank among the leg's pages.
+
+ A pageless file takes the rank of the page below it, so pages fuse as they
+ would without it. Items may be results or ``(result, extra)`` pairs.
+ """
+ ranked: list[tuple[int, Any]] = []
+ pages = 0
+ for item in results:
+ result = item[0] if isinstance(item, tuple) else item
+ ranked.append((pages, item))
+ if result.page_type != PAGELESS_FILE:
+ pages += 1
+ return ranked
+
+
+def _hit_dict_from_result(result: Any) -> dict:
+ """Convert a retriever result object to the pipeline's dict shape."""
+ return {
+ "page_id": result.page_id,
+ "title": getattr(result, "title", ""),
+ "snippet": getattr(result, "snippet", ""),
+ "page_type": getattr(result, "page_type", ""),
+ "score": 0.0,
+ "_sources": set(),
+ }
+
+
+__all__ = [
+ "_CONCEPT_FETCH_LIMIT",
+ "_DECISION_OVERFETCH",
+ "_DECISION_PREFIX",
+ "_FILENAME_LEG_RRF_K",
+ "_GRAPH_EXPAND_DAMPING",
+ "_GRAPH_EXPAND_MAX_NEW",
+ "_GRAPH_EXPAND_TOP_N",
+ "_HUB_DEGREE_FLOOR",
+ "_HUB_DEGREE_PERCENTILE",
+ "_LEG_RECORD",
+ "_PAGERANK_BIAS_MAX",
+ "_PARENT_EXPAND_BOOST",
+ "_PARENT_EXPAND_TOP_N",
+ "_PARENT_MIN_SHARE",
+ "_QUESTION_VECTORS",
+ "_QUESTION_VECTOR_CACHE_MAX",
+ "_RETRIEVAL_FETCH_LIMIT",
+ "_RRF_K",
+ "_RRF_SCORE_SCALE",
+ "_SUBSYSTEM_QUERY_RE",
+ "_SYMBOL_LEG_FETCH_LIMIT",
+ "_SYMBOL_LEG_MAX_PAGES",
+ "_SYMBOL_LEG_RRF_K",
+ "_TEST_QUERY_RE",
+ "_common_ancestor",
+ "_hub_degree_cutoff",
+ "_projected_edges",
+ "_projected_pair",
+ "_record_leg",
+ "_safe_filename_search",
+ "_safe_fts_search",
+ "_safe_symbol_search",
+ "_safe_vector_search",
+ "_semantic_concept_paths",
+ "apply_pagerank_bias",
+ "begin_leg_record",
+ "degraded_legs",
+ "demote_noise_hits",
+ "expand_via_graph",
+ "expand_via_parent_page",
+ "filename_backed_pages",
+ "hybrid_retrieve",
+ "hydrate_hits",
+ "is_subsystem_query",
+ "leg_ranks",
+ "question_vector",
+ "retrieval_legs",
+ "stamp_hybrid_rank",
+ "vector_search",
+]
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.py
new file mode 100644
index 000000000..cb843bf9e
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/graph_expand.py
@@ -0,0 +1,199 @@
+"""Graph expansion: 1-hop neighbours of the top hits join the candidate set."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy import select
+
+from repowise.core.ingestion.models import CONTAINMENT_EDGE_TYPES
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.models import GraphEdge, GraphNode, Page
+from repowise.server.mcp_server._graph_files import (
+ is_symbol_node,
+ keep_projected_edge,
+ node_to_file,
+ per_index,
+)
+from repowise.server.mcp_server._page_paths import PAGELESS_FILE
+
+# Seeds to expand from, and the most files expansion may add.
+_GRAPH_EXPAND_TOP_N = 2
+_GRAPH_EXPAND_MAX_NEW = 3
+
+# A neighbour above max(floor, p99) degree is a hub: near everything, so it
+# says nothing about this question, yet PageRank would rank it first.
+_HUB_DEGREE_FLOOR = 50
+_HUB_DEGREE_PERCENTILE = 0.99
+
+# Expanded hits score below their parent but can pass the tail of the top 5.
+_GRAPH_EXPAND_DAMPING = 0.7
+
+
+def _projected_pair(
+ src: str | None, tgt: str | None, edge_type: str | None, confidence: float | None
+) -> tuple[str | None, str | None]:
+ """An edge's two endpoints as file paths, or ``(None, None)`` to drop it.
+
+ Only symbol-layer edges are guarded; file-layer edges such as a module's
+ ``co_changes`` with its docs pass as they are.
+ """
+ src = src or ""
+ tgt = tgt or ""
+ if not is_symbol_node(src) and not is_symbol_node(tgt):
+ return (src or None, tgt or None)
+ src_file, tgt_file = node_to_file(src), node_to_file(tgt)
+ if not keep_projected_edge(src_file, tgt_file, edge_type, confidence):
+ return (None, None)
+ return (src_file, tgt_file)
+
+
+async def _projected_edges(session: Any, repo_id: str) -> list[tuple[str, str]]:
+ """Every non-containment edge for *repo_id*, as a pair of file paths.
+
+ One scan projected in Python: a symbol-layer prefix match cannot use the
+ node-id index. Containment always projects to a self-loop.
+ """
+ res = await session.execute(
+ select(
+ GraphEdge.source_node_id,
+ GraphEdge.target_node_id,
+ GraphEdge.edge_type,
+ GraphEdge.confidence,
+ ).where(
+ GraphEdge.repository_id == repo_id,
+ GraphEdge.edge_type.notin_(CONTAINMENT_EDGE_TYPES),
+ )
+ )
+ pairs: list[tuple[str, str]] = []
+ for src, tgt, etype, conf in res.all():
+ src_file, tgt_file = _projected_pair(src, tgt, etype, conf)
+ if src_file and tgt_file:
+ pairs.append((src_file, tgt_file))
+ return pairs
+
+
+def _hub_degree_cutoff(degree: dict[str, int]) -> int:
+ """Degree at which a candidate counts as a hub: ``max(floor, p99)`` of the set."""
+ if not degree:
+ return 1 << 30
+ values = sorted(degree.values())
+ idx = min(len(values) - 1, int(len(values) * _HUB_DEGREE_PERCENTILE))
+ return max(_HUB_DEGREE_FLOOR, values[idx])
+
+
+def _seed_neighbors(
+ pairs: list[tuple[str, str]], seed_set: set[str], existing: set[Any]
+) -> tuple[set[str], dict[str, int]]:
+ """Files one hop from a seed and not yet a candidate, plus every file's degree."""
+ neighbors: set[str] = set()
+ degree: dict[str, int] = {}
+ for src_file, tgt_file in pairs:
+ degree[src_file] = degree.get(src_file, 0) + 1
+ degree[tgt_file] = degree.get(tgt_file, 0) + 1
+ for mine, theirs in ((src_file, tgt_file), (tgt_file, src_file)):
+ if mine in seed_set and theirs not in existing:
+ neighbors.add(theirs)
+ return neighbors, degree
+
+
+async def _neighbor_pages(
+ session: Any, neighbors: set[str]
+) -> tuple[list[Any], dict[str, float]]:
+ """File-page rows of *neighbors*, and each neighbour's PageRank."""
+ page_res = await session.execute(
+ select(Page.target_path, Page.summary, Page.page_type).where(
+ Page.target_path.in_(sorted(neighbors)),
+ Page.page_type == "file_page",
+ )
+ )
+ page_rows = list(page_res.all())
+
+ pr_res = await session.execute(
+ select(GraphNode.node_id, GraphNode.pagerank).where(
+ GraphNode.node_id.in_(sorted(neighbors)),
+ GraphNode.node_type == "file",
+ )
+ )
+ pr_by_path = {row[0]: float(row[1] or 0.0) for row in pr_res.all()}
+ return page_rows, pr_by_path
+
+
+def _drop_hubs(page_rows: list[Any], degree: dict[str, int]) -> list[Any]:
+ """*page_rows* without hubs, or all of them when every row is a hub."""
+ # Cutoff over the candidates on offer, not the whole repo.
+ cutoff = _hub_degree_cutoff({row[0]: degree.get(row[0], 0) for row in page_rows})
+ non_hub = [row for row in page_rows if degree.get(row[0], 0) <= cutoff]
+ return non_hub or page_rows
+
+
+def _expansion_candidates(
+ seeds: list[dict], page_rows: list[Any], pr_by_path: dict[str, float]
+) -> list[dict]:
+ """One damped hit per neighbour page, most central first."""
+ strongest_parent = max(
+ seeds, key=lambda hit: hit.get("score", 0.0), default={}
+ )
+ parent_score = strongest_parent.get("score", 0.0)
+ confidence_factor = strongest_parent.get("_confidence_score_factor")
+ candidates: list[dict] = []
+ for path, summary, page_type in page_rows:
+ candidate = {
+ "page_id": f"file_page:{path}",
+ "target_path": path,
+ "title": f"File: {path}",
+ "summary": summary or "",
+ "snippet": (summary or "")[:200],
+ "page_type": page_type or "file_page",
+ "score": parent_score * _GRAPH_EXPAND_DAMPING,
+ "_sources": {"graph_expand"},
+ "_expanded_from": "graph",
+ "_pagerank": pr_by_path.get(path, 0.0),
+ }
+ if confidence_factor is not None:
+ candidate["_confidence_score_factor"] = confidence_factor
+ candidates.append(candidate)
+
+ # Path breaks PageRank ties so the pick does not follow database row order.
+ candidates.sort(key=lambda c: (-c.get("_pagerank", 0.0), c["target_path"]))
+ return candidates
+
+
+async def expand_via_graph(hits: list[dict], ctx: Any, repo_id: str) -> list[dict]:
+ """Add up to ``_GRAPH_EXPAND_MAX_NEW`` one-hop neighbours of the top hits.
+
+ Recovers the right file when retrieval landed next to it (a consumer
+ instead of the implementation). Only neighbours with a file page that are
+ not yet candidates join, marked ``_expanded_from`` with a damped score.
+ """
+ if not hits:
+ return hits
+ # Seeded from pages only, so a pageless file cannot change what pages add.
+ seeds = [h for h in hits if h.get("page_type") != PAGELESS_FILE][:_GRAPH_EXPAND_TOP_N]
+ seed_paths = [h.get("target_path") for h in seeds if h.get("target_path")]
+ if not seed_paths:
+ return hits
+ existing = {h.get("target_path") for h in hits}
+
+ async with get_session(ctx.session_factory) as session:
+ # Every edge type, co-changes included, projected from the symbol layer
+ # too so call edges reach the seeds.
+ pairs = await per_index(
+ session, repo_id, "projected_edges", lambda: _projected_edges(session, repo_id)
+ )
+ neighbors, degree = _seed_neighbors(pairs, set(seed_paths), existing)
+ if not neighbors:
+ return hits
+ page_rows, pr_by_path = await _neighbor_pages(session, neighbors)
+
+ if not page_rows:
+ return hits
+
+ page_rows = _drop_hubs(page_rows, degree)
+ additions = _expansion_candidates(seeds, page_rows, pr_by_path)[:_GRAPH_EXPAND_MAX_NEW]
+ if not additions:
+ return hits
+
+ combined = hits + additions
+ combined.sort(key=lambda h: h["score"], reverse=True)
+ return combined
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.py
new file mode 100644
index 000000000..00926a8a8
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/hydrate.py
@@ -0,0 +1,143 @@
+"""Hydrate hits with Page metadata, then bias them by PageRank."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from sqlalchemy import select
+
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.models import GraphNode, Page
+from repowise.server.mcp_server._page_paths import PAGELESS_FILE, pageless_path
+
+_log = logging.getLogger("repowise.mcp.answer")
+
+# Capped multiplicative bias, [1.0, 1.3]: breaks ties, never beats a strong match.
+_PAGERANK_BIAS_MAX = 0.3
+
+
+async def _page_meta_by_id(ctx: Any, page_ids: list[str]) -> dict[str, dict[str, str]]:
+ """Path, summary, type and freshness of each page id that has a page row."""
+ async with get_session(ctx.session_factory) as session:
+ res = await session.execute(
+ select(
+ Page.id,
+ Page.target_path,
+ Page.summary,
+ Page.page_type,
+ Page.freshness_status,
+ ).where(Page.id.in_(page_ids))
+ )
+ return {
+ row[0]: {
+ "target_path": row[1] or "",
+ "summary": row[2] or "",
+ "page_type": row[3] or "",
+ "freshness": row[4] or "",
+ }
+ for row in res.all()
+ }
+
+
+def _hit_meta(hit: dict, meta_by_id: dict[str, dict[str, str]]) -> dict[str, str] | None:
+ """The page metadata behind *hit*, or None when there is nothing to cite."""
+ meta = meta_by_id.get(hit["page_id"])
+ if meta is None and hit.get("page_type") == PAGELESS_FILE:
+ # A pageless file: its path is in its id, and it serves matched code.
+ meta = {"target_path": pageless_path(hit["page_id"]), "page_type": PAGELESS_FILE}
+ return meta
+
+
+def _apply_meta(hit: dict, meta: dict[str, str], scope: str | None) -> bool:
+ """Copy *meta* onto *hit*; False when the hit is tombstoned or out of scope."""
+ # Tombstoned pages describe deleted or renamed files.
+ if meta.get("freshness") == "tombstone":
+ return False
+ target_path = meta.get("target_path", "")
+ if scope and target_path and not target_path.startswith(scope):
+ return False
+ hit["target_path"] = target_path
+ hit["summary"] = meta.get("summary", "")
+ # The page table's type wins; retrievers can carry stale ones.
+ hit["page_type"] = meta.get("page_type") or hit.get("page_type", "")
+ return True
+
+
+async def hydrate_hits(hits: list[dict], ctx: Any, *, scope: str | None = None) -> list[dict]:
+ """Attach path, summary and page type in place.
+
+ Drops hits with no page, tombstoned pages and paths outside ``scope``.
+ """
+ if not hits:
+ return hits
+ meta_by_id = await _page_meta_by_id(ctx, [h["page_id"] for h in hits])
+
+ out: list[dict] = []
+ pageless = 0
+ for h in hits:
+ meta = _hit_meta(h, meta_by_id)
+ if meta is None:
+ # No page behind the id (a decision vector or store drift): nothing
+ # to cite, and the later gates are keyed on a path it lacks.
+ pageless += 1
+ continue
+ if _apply_meta(h, meta, scope):
+ out.append(h)
+ if pageless:
+ _log.warning(
+ "get_answer dropped %d of %d retrieved hit(s) with no page row behind them; "
+ "an id in a retriever that the page table does not know is either a "
+ "non-page vector or a three-store disagreement (repowise doctor --repair)",
+ pageless,
+ len(hits),
+ )
+ return out
+
+
+async def _file_pageranks(ctx: Any, paths: list[str]) -> dict[str, float]:
+ """PageRank of each file node among *paths*."""
+ async with get_session(ctx.session_factory) as session:
+ res = await session.execute(
+ select(GraphNode.node_id, GraphNode.pagerank).where(
+ GraphNode.node_id.in_(paths),
+ GraphNode.node_type == "file",
+ )
+ )
+ return {row[0]: float(row[1] or 0.0) for row in res.all()}
+
+
+def _max_page_rank(hits: list[dict], pr_by_path: dict[str, float]) -> float:
+ """The PageRank the bias normalises against."""
+ # Over pages only, so a pageless file cannot change their bias.
+ paged = {h.get("target_path") for h in hits if h.get("page_type") != PAGELESS_FILE}
+ max_pr = max((pr for path, pr in pr_by_path.items() if path in paged), default=0.0)
+ return max_pr or max(pr_by_path.values(), default=0.0)
+
+
+async def apply_pagerank_bias(hits: list[dict], ctx: Any) -> None:
+ """Multiply each hit's score by a PageRank bias, then re-sort (in place).
+
+ Normalised within the candidates, not the whole graph, so a famous but
+ unrelated file gets no extra reward.
+ """
+ if not hits:
+ return
+ paths = [h.get("target_path") for h in hits if h.get("target_path")]
+ if not paths:
+ return
+ pr_by_path = await _file_pageranks(ctx, paths)
+ if not pr_by_path:
+ return
+ max_pr = _max_page_rank(hits, pr_by_path)
+ if max_pr <= 0:
+ return
+
+ for h in hits:
+ pr = pr_by_path.get(h.get("target_path"), 0.0)
+ bias = 1.0 + _PAGERANK_BIAS_MAX * min(pr / max_pr, 1.0)
+ h["_pagerank"] = pr
+ h["_pagerank_bias"] = round(bias, 3)
+ h["score"] = h.get("score", 0.0) * bias
+
+ hits.sort(key=lambda h: h["score"], reverse=True)
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.py
new file mode 100644
index 000000000..f262b392e
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/leg_record.py
@@ -0,0 +1,43 @@
+"""Which retrieval legs ran for the current question, and how each ended."""
+
+from __future__ import annotations
+
+import contextvars
+
+# How each leg ended for the current question. Legs fail soft, so without
+# this a lexical-only answer looks like a whole one; ``embedder_live`` says an
+# embedder exists, this says whether it was used.
+_LEG_RECORD: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar(
+ "repowise_retrieval_legs", default=None
+)
+
+
+def _record_leg(leg: str, outcome: str) -> None:
+ """Note how one retrieval leg ended, if anyone is collecting."""
+ record = _LEG_RECORD.get()
+ if record is not None:
+ record[leg] = outcome
+
+
+def begin_leg_record() -> dict[str, str]:
+ """Start collecting leg outcomes for one question. Returns the record.
+
+ ``asyncio.gather`` copies the context, not the dict, so every leg task
+ writes into this one record.
+ """
+ record: dict[str, str] = {}
+ _LEG_RECORD.set(record)
+ return record
+
+
+def retrieval_legs() -> dict[str, str]:
+ """Leg outcomes for the last question, e.g. ``{"fts": "ok", "vector": "timeout"}``.
+
+ Holds an ``embed`` key when embedding the question failed.
+ """
+ return dict(_LEG_RECORD.get() or {})
+
+
+def degraded_legs(legs: dict[str, str]) -> list[str]:
+ """The legs that broke, sorted. ``keyless`` is configuration, not breakage."""
+ return sorted(leg for leg, outcome in legs.items() if outcome not in ("ok", "keyless"))
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.py
new file mode 100644
index 000000000..ef4b4b93d
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/noise.py
@@ -0,0 +1,35 @@
+"""Noise demotion: decision records and test file pages sink below real hits."""
+
+from __future__ import annotations
+
+import re
+
+from repowise.core.test_paths import is_test_path
+from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES
+
+# Same test-path rules as search_codebase's demotion, on the answer hit shape.
+_TEST_QUERY_RE = re.compile(
+ r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",
+ re.IGNORECASE,
+)
+
+
+def demote_noise_hits(hits: list[dict], question: str, *, is_why: bool) -> list[dict]:
+ """Stable-partition noise below real hits before the top-5 cap; nothing is dropped.
+
+ Decision records are noise unless the question is why-shaped, test files
+ unless it is about tests. Test support such as ``conftest.py`` is not noise.
+ """
+ if not hits:
+ return hits
+ test_focused = bool(_TEST_QUERY_RE.search(question))
+
+ def _is_noise(h: dict) -> bool:
+ pt = h.get("page_type")
+ return (pt == "decision_record" and not is_why) or (
+ pt in FILE_ROW_TYPES and not test_focused and is_test_path(h.get("target_path") or "")
+ )
+
+ real = [h for h in hits if not _is_noise(h)]
+ noise = [h for h in hits if _is_noise(h)]
+ return real + noise
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.py
new file mode 100644
index 000000000..b7ed0c61d
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/parent_page.py
@@ -0,0 +1,215 @@
+"""Parent-concept surfacing: a subsystem-shaped question leads with its concept page.
+
+Retrieval ranks a subsystem's member files and child pages above the concept
+page for the whole subsystem, because they match the question's noun more
+tightly. Two signals pick the page, neither tuned to a repository: a tight
+cluster of hits in one directory (structural), else a concept-restricted vector
+search (semantic). Every other question keeps its ranking.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any
+
+from sqlalchemy import select
+
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.models import Page
+from repowise.server.mcp_server._answer_pipeline.vectors import question_vector, vector_search
+from repowise.server.mcp_server._helpers import vector_search_timeout_s
+
+# High precision on purpose: each phrase is anchored to an overview intent,
+# because fragments like "parts of" or "which module" fill ordinary questions.
+_SUBSYSTEM_QUERY_RE = re.compile(
+ r"\boverview of\b|\bgive me an overview\b|\bhigh[- ]level\b|"
+ r"\bmain parts of\b|\bwhat (?:are|is) the (?:main )?(?:parts|pieces|components) of\b|"
+ r"\barchitecture of\b|\bwalk me through\b|"
+ r"\bhow (?:is|are|does) .+ (?:organi[sz]ed|structured|laid out|put together)\b|"
+ r"\bwhat subsystem\b|\bwhich subsystem\b|"
+ r"\bwhere would i (?:add|put)\b|\bwhere do i (?:add|put)\b",
+ re.IGNORECASE,
+)
+
+# Strongest real hits (no decision records, no pathless pages) to cluster.
+_PARENT_EXPAND_TOP_N = 8
+# Hits a directory must directly hold to count as a tight cluster.
+_PARENT_MIN_SHARE = 2
+# Just past the top score: leads without reading as a dominant retrieval.
+_PARENT_EXPAND_BOOST = 1.02
+# Wide, since concept pages are a small minority of the vector store.
+_CONCEPT_FETCH_LIMIT = 60
+
+
+def is_subsystem_query(question: str) -> bool:
+ """True when the question asks about a subsystem as a whole."""
+ return bool(_SUBSYSTEM_QUERY_RE.search(question or ""))
+
+
+def _common_ancestor(paths: set[str]) -> str:
+ """Longest shared directory prefix of the given paths, segment-wise."""
+ split = [p.split("/") for p in paths]
+ common: list[str] = []
+ for segs in zip(*split, strict=False):
+ if len(set(segs)) == 1:
+ common.append(segs[0])
+ else:
+ break
+ return "/".join(common)
+
+
+async def _semantic_concept_paths(question: str, ctx: Any) -> list[str]:
+ """Concept/layer page paths a concept-restricted vector search ranks first; [] on error."""
+ vs = getattr(ctx, "vector_store", None)
+ if vs is None:
+ return []
+ try:
+ results = await asyncio.wait_for(
+ vector_search(
+ vs, question, _CONCEPT_FETCH_LIMIT, vector=await question_vector(ctx, question)
+ ),
+ timeout=vector_search_timeout_s(),
+ )
+ except Exception:
+ return []
+ out: list[str] = []
+ for r in results:
+ if getattr(r, "page_type", "") not in ("module_page", "layer_page"):
+ continue
+ tp = _result_target_path(r)
+ if tp and tp not in out:
+ out.append(tp)
+ return out
+
+
+def _result_target_path(result: Any) -> str:
+ """A vector hit's target path, read from its page id when it carries none."""
+ tp = getattr(result, "target_path", "") or ""
+ if not tp:
+ pid = getattr(result, "page_id", "") or ""
+ tp = pid.split(":", 1)[1] if ":" in pid else pid
+ return tp
+
+
+def _cluster_parents(top: list[dict]) -> tuple[dict[str, int], dict[str, set[str]]]:
+ """Hits directly in each directory, and the hits under each ancestor."""
+ imm_count: dict[str, int] = {}
+ covers: dict[str, set[str]] = {}
+ for h in top:
+ tp = h["target_path"].rstrip("/")
+ parent = tp.rsplit("/", 1)[0] if "/" in tp else ""
+ if parent:
+ imm_count[parent] = imm_count.get(parent, 0) + 1
+ anc = parent
+ while anc:
+ covers.setdefault(anc, set()).add(tp)
+ anc = anc.rsplit("/", 1)[0] if "/" in anc else ""
+ return imm_count, covers
+
+
+async def _concept_pages(ctx: Any, candidates: set[str]) -> list[Any]:
+ """Concept/layer page rows whose target path is one of *candidates*."""
+ async with get_session(ctx.session_factory) as session:
+ return (
+ await session.execute(
+ select(Page.target_path, Page.title, Page.summary, Page.page_type).where(
+ Page.target_path.in_(candidates),
+ Page.page_type.in_(("module_page", "layer_page")),
+ )
+ )
+ ).all()
+
+
+def _tight_cluster_page(
+ tight_pages: set[str],
+ by_path: dict[str, Any],
+ imm_count: dict[str, int],
+ covers: dict[str, set[str]],
+) -> Any:
+ """The tightest cluster's page; sibling clusters roll up to their common ancestor."""
+ rollup = _common_ancestor(tight_pages)
+ if len(tight_pages) > 1 and rollup in by_path and rollup in covers:
+ return by_path[rollup]
+ best_tp = max(tight_pages, key=lambda d: (imm_count[d], d.count("/")))
+ return by_path[best_tp]
+
+
+def _semantic_or_cover_page(
+ semantic_paths: list[str], by_path: dict[str, Any], covers: dict[str, set[str]]
+) -> Any:
+ """The semantically closest concept page, else the deepest covering ancestor."""
+ winner = next((by_path[tp] for tp in semantic_paths if tp in by_path), None)
+ if winner is None:
+ cov_pages = [a for a in covers if a in by_path]
+ if cov_pages:
+ winner = by_path[max(cov_pages, key=lambda a: (a.count("/"), len(covers[a])))]
+ return winner
+
+
+def _lead_with_parent(hits: list[dict], winner: Any) -> list[dict]:
+ """Promote *winner* to lead if already retrieved, else inject it as the lead."""
+ best_tp, best_title, best_summary, best_pt = winner
+ top_score = max((h.get("score", 0.0) for h in hits), default=0.0) or 1.0
+ lead_score = top_score * _PARENT_EXPAND_BOOST
+
+ retrieved = next((h for h in hits if h.get("target_path") == best_tp), None)
+ if retrieved is not None:
+ if retrieved.get("score", 0.0) < lead_score:
+ retrieved["score"] = lead_score
+ src = retrieved.get("_sources")
+ if isinstance(src, set):
+ src.add("parent_promote")
+ hits.sort(key=lambda x: x.get("score", 0.0), reverse=True)
+ return hits
+
+ parent_hit = {
+ "page_id": f"{best_pt}:{best_tp}",
+ "target_path": best_tp,
+ "title": best_title or f"Overview: {best_tp}",
+ "summary": best_summary or "",
+ "snippet": (best_summary or "")[:200],
+ "page_type": best_pt or "module_page",
+ "score": lead_score,
+ "_sources": {"parent_expand"},
+ "_expanded_from": "parent",
+ }
+ combined = [parent_hit, *hits]
+ combined.sort(key=lambda h: h["score"], reverse=True)
+ return combined
+
+
+async def expand_via_parent_page(hits: list[dict], question: str, ctx: Any) -> list[dict]:
+ """Lead a subsystem-shaped question with the concept page for its subsystem.
+
+ A tight structural cluster wins over the semantic guess. The page is
+ promoted in place when already retrieved, else injected as the lead.
+ """
+ if not hits or not is_subsystem_query(question):
+ return hits
+ top = [h for h in hits if h.get("target_path") and h.get("page_type") != "decision_record"][
+ :_PARENT_EXPAND_TOP_N
+ ]
+ if not top:
+ return hits
+
+ imm_count, covers = _cluster_parents(top)
+ tight_clusters = {d for d, c in imm_count.items() if c >= _PARENT_MIN_SHARE}
+ semantic_paths = await _semantic_concept_paths(question, ctx)
+ candidates = set(covers) | set(imm_count) | set(semantic_paths)
+ if not candidates:
+ return hits
+
+ rows = await _concept_pages(ctx, candidates)
+ if not rows:
+ return hits
+ by_path = {r[0]: r for r in rows}
+
+ tight_pages = tight_clusters & by_path.keys()
+ if tight_pages:
+ winner = _tight_cluster_page(tight_pages, by_path, imm_count, covers)
+ else:
+ winner = _semantic_or_cover_page(semantic_paths, by_path, covers)
+ if winner is None:
+ return hits
+ return _lead_with_parent(hits, winner)
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.py b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.py
new file mode 100644
index 000000000..b50ddff5d
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/_answer_pipeline/vectors.py
@@ -0,0 +1,88 @@
+"""The question's embedding, computed once per call, and the shared vector read."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import OrderedDict
+from typing import Any
+
+from repowise.core.providers.embedding import store_has_semantic_vectors
+from repowise.server.mcp_server._answer_pipeline.leg_record import _record_leg
+from repowise.server.mcp_server._helpers import _EMBED_TIMEOUT_ENV, embed_timeout_s
+
+_log = logging.getLogger("repowise.mcp.answer")
+
+# One embedding per (store, question), shared by every stage that needs it.
+# The key holds the store in its value so a recycled id cannot match. A few
+# entries so concurrent questions do not evict each other.
+_QUESTION_VECTOR_CACHE_MAX = 4
+_QUESTION_VECTORS: OrderedDict[tuple[int, str], tuple[Any, list[float]]] = OrderedDict()
+
+
+async def question_vector(ctx: Any, question: str) -> list[float] | None:
+ """The question's embedding, cached per store and question.
+
+ None when there is no store or embedder, or the embed failed; callers then
+ search by text, which costs an embed but is still correct.
+ """
+ store = getattr(ctx, "vector_store", None)
+ if store is None or not question:
+ return None
+ if not store_has_semantic_vectors(store):
+ return None
+
+ key = (id(store), question)
+ cached = _QUESTION_VECTORS.get(key)
+ if cached is not None:
+ _QUESTION_VECTORS.move_to_end(key)
+ return cached[1]
+
+ timeout_s = embed_timeout_s()
+ try:
+ vectors = await asyncio.wait_for(store.embed_texts([question]), timeout=timeout_s)
+ except TimeoutError:
+ # Recorded so a lexical-only answer is visible as one.
+ _record_leg("embed", "timeout")
+ _log.warning(
+ "get_answer could not embed the question within %.1fs; retrieval "
+ "continues without a question vector. Raise it with %s=<seconds>.",
+ timeout_s,
+ _EMBED_TIMEOUT_ENV,
+ )
+ return None
+ except Exception:
+ _record_leg("embed", "error")
+ _log.warning(
+ "get_answer could not embed the question up front; each retrieval "
+ "stage will embed it again",
+ exc_info=True,
+ )
+ return None
+ if not vectors:
+ # Backend without its own embedder: each stage embeds the text itself.
+ _log.debug("Vector store cannot embed directly; per-stage embedding stands")
+ return None
+
+ vector = [float(v) for v in vectors[0]]
+ _QUESTION_VECTORS[key] = (store, vector)
+ while len(_QUESTION_VECTORS) > _QUESTION_VECTOR_CACHE_MAX:
+ _QUESTION_VECTORS.popitem(last=False)
+ return vector
+
+
+async def vector_search(
+ store: Any, question: str, limit: int, *, vector: list[float] | None
+) -> list[Any]:
+ """Nearest pages to *question*, by *vector* when the backend supports it.
+
+ [] on a keyless store; checked here because every vector read in the
+ pipeline goes through this, and the text fallback would embed.
+ """
+ if not store_has_semantic_vectors(store):
+ return []
+ if vector is not None:
+ by_vector = await store.search_by_vector(vector, limit=limit)
+ if by_vector is not None:
+ return by_vector
+ return await store.search(question, limit=limit)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.py
index 748c4929e..791895736 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_search.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py
@@ -7,6 +7,7 @@ import contextlib
import logging
import re
from collections.abc import Container, Sequence
+from dataclasses import dataclass
from typing import Any
from sqlalchemy import select
@@ -1053,6 +1054,246 @@ def _missing_named_symbols(
return missing
+@dataclass(frozen=True)
+class _SearchPlan:
+ """One structured search: the caller's arguments plus the derived query."""
+
+ query: str
+ limit: int
+ page_type: str | None
+ kind: str | None
+ symbol_kind: str | None
+ mode: str
+ names: Container[str] | None
+ canonical_symbol: tuple[Any, str] | None
+ symbol_query: str
+ fetch: int
+ multi: bool
+
+
+@dataclass
+class _Pools:
+ """Each row kind merged and ranked across the repos searched."""
+
+ contexts: list[Any]
+ symbols: list[dict]
+ spare: list[dict]
+ files: list[dict]
+ indexed_paths: list[str]
+ concepts: list[dict]
+
+
+def _symbol_query_for(
+ query: str, mode: str, names: Container[str] | None, canonical_symbol: Any
+) -> str:
+ """The text the symbol scorer ranks on."""
+ # Scored on the identifiers, not the prose: words like "is" would match
+ # ``is_ci`` and bury the symbol the question names.
+ symbol_query = query
+ if canonical_symbol:
+ symbol_query = canonical_symbol[1]
+ if mode == "hybrid":
+ _idents = _embedded_identifiers(query, names)
+ # No identifier: content words, else the raw text.
+ symbol_query = (
+ " ".join(_idents) if _idents else " ".join(content_terms(query)) or query
+ )
+ return symbol_query
+
+
+def _symbol_fetch(mode: str, limit: int) -> int:
+ """How many symbol rows each repo fetches."""
+ # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every
+ # candidate so ``fuzzy_omitted`` can count what an exact hit hides.
+ fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit
+ if mode == "symbol":
+ fetch = _MAX_CANDIDATES
+ return fetch
+
+
+async def _search_one_repo(
+ ctx: Any, plan: _SearchPlan
+) -> tuple[list[dict], list[dict], list[str], list[dict]]:
+ """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""
+ s: list[dict] = []
+ f: list[dict] = []
+ paths: list[str] = []
+ c: list[dict] = []
+ if plan.mode in ("symbol", "hybrid"):
+ s = await search_symbols_single(
+ ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind
+ )
+ if plan.mode == "path":
+ f = await search_paths_single(ctx, plan.query, plan.limit)
+ async with get_session(ctx.session_factory) as session:
+ paths = (await file_path_index(session, (await _get_repo(session)).id)).paths
+ if plan.mode == "hybrid":
+ c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)
+ for item in c:
+ item["type"] = "page"
+ await attach_hit_symbols(ctx, plan.query, c)
+ for rows in (s, f, c):
+ _tag_repo(rows, ctx, plan.multi)
+ return s, f, paths, c
+
+
+def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:
+ """Merge each repo's rows into one ranked pool per row kind."""
+ parts = [part for _, part in answered]
+ return _Pools(
+ contexts=[ctx for ctx, _ in answered],
+ symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),
+ # Rows past each repo's limit only backfill slots that collapsing frees.
+ spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),
+ files=merge_ranked((p[1] for p in parts), _neg_score("score")),
+ indexed_paths=[path for p in parts for path in p[2]],
+ concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),
+ )
+
+
+def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:
+ """Keep exact name and path matches ahead of their fuzzy neighbours."""
+ if plan.mode == "symbol":
+ pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)
+ elif plan.mode == "hybrid" and candidates:
+ pools.symbols = _protect_named_symbols(candidates, pools.symbols)
+ if plan.mode == "path":
+ pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)
+
+
+def _symbol_mode_results(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
+) -> tuple[list[dict], int]:
+ """Symbol mode rows, and how many fuzzy names an exact hit hides."""
+ # An exact hit is the answer; its fuzzy neighbours are only counted.
+ fuzzy_omitted = 0
+ pool = pools.symbols + pools.spare
+ named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []
+ if named:
+ canonical_symbol = plan.canonical_symbol
+ ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()
+ shown = {id(s) for s in named}
+ fuzzy_omitted = sum(
+ 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()
+ )
+ return (named or pools.symbols)[: plan.limit], fuzzy_omitted
+
+
+def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:
+ """Hybrid rows: symbol matches interleaved with concept pages for new files."""
+ window = _hybrid_window(
+ plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names
+ )
+ # One row per file; freed slots fill with the next pages, then symbols.
+ served = {id(item) for item in window}
+ rest = [
+ item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served
+ ]
+ results = _collapse_by_file(window + rest)[: plan.limit]
+ # Non-file pages leave the window unrefilled unless the caller asked for
+ # pages by type or kind="doc".
+ if not plan.page_type and plan.kind != "doc":
+ results = [item for item in results if hit_file_path(item)]
+ return results
+
+
+def _select_results(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
+) -> tuple[list[dict], int]:
+ """The served rows for the plan's mode, and the symbol-mode fuzzy count."""
+ if plan.mode == "symbol":
+ return _symbol_mode_results(plan, pools, candidates, exact)
+ if plan.mode == "path":
+ return pools.files[: plan.limit], 0
+ return _hybrid_mode_results(plan, pools, exact), 0
+
+
+async def _single_repository(contexts: list[Any], multi: bool) -> Any:
+ """The repository row a single-repo search ran against, else None."""
+ if multi or not contexts:
+ return None
+ async with get_session(contexts[0].session_factory) as session:
+ return await _get_repo(session)
+
+
+def _exact_match_note(
+ candidates: list[str],
+ exact: bool,
+ missing: list[str],
+ fuzzy_omitted: int,
+ has_results: bool,
+ grep_hint: str | None,
+) -> str | None:
+ """The note qualifying an identifier query's results, if one is owed."""
+ if missing:
+ shown = ", ".join(repr(c) for c in missing[:3])
+ # An empty window's grep_hint already carries the sweep advice.
+ pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""
+ sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT
+ return (
+ f"No indexed symbol is named {shown}, so no symbol is returned "
+ f"for it.{pages} Recheck the spelling, or search a shorter part "
+ f"of the name.{sweep}"
+ )
+ if not exact:
+ shown = ", ".join(repr(c) for c in candidates[:3])
+ return (
+ f"No indexed symbol exactly matches {shown}. The results are "
+ "fuzzy neighbours ranked by token overlap — confirm a hit names "
+ "what you meant before relying on it. If you expected an exact "
+ "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT
+ )
+ if fuzzy_omitted:
+ return (
+ f"{fuzzy_omitted} other symbols contain this name; search a "
+ "longer or partial name to list them."
+ )
+ return None
+
+
+def _match_named(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str]
+) -> tuple[bool, list[str]]:
+ """Whether a returned symbol names the query exactly, and the names none does.
+
+ Missing names leave only exact symbols in *pools*.
+ """
+ exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False
+ missing = _missing_named_symbols(
+ candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts
+ )
+ if missing:
+ pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]
+ pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]
+ return exact, missing
+
+
+def _attach_match_notes(
+ response: dict,
+ candidates: list[str],
+ exact: bool,
+ missing: list[str],
+ grep_hint: str | None,
+) -> None:
+ """Exact-match flag, its note, and the grep fallback for an empty window."""
+ results = response["results"]
+ # Fuzzy neighbours of a missing name look authoritative, so say so.
+ if candidates:
+ response["exact_match"] = exact and not missing
+ note = _exact_match_note(
+ candidates,
+ exact,
+ missing,
+ response.get("fuzzy_omitted", 0),
+ bool(results),
+ grep_hint,
+ )
+ if note:
+ response["note"] = note
+ if grep_hint and not results:
+ response["grep_hint"] = grep_hint
+
+
async def _structured_search(
query: str,
limit: int,
@@ -1065,128 +1306,36 @@ async def _structured_search(
names: Container[str] | None = None,
fed: Federation | None = None,
) -> dict:
- """Run symbol / path / hybrid search and shape the response.
+ """Run symbol / path / hybrid search, across repos when federated, and shape the response.
- Honours ``repo="all"`` (federates across contexts, then re-ranks by score),
- per-repo ``exclude_patterns`` and tombstones (enforced inside the
- single-repo helpers). The grep_hint is attached only as a fallback — when
- the structural index produced nothing, the agent still has a path forward.
+ ``grep_hint`` is attached only when nothing was found.
"""
multi = fed is not None and fed.multi
-
- # A hybrid query is prose wrapped around an identifier ("where is X
- # defined"). The symbol scorer ranks on token overlap, so handing it the
- # raw prose lets stopword-ish tokens ("is" -> is_ci, "filter" ->
- # FilterRegistry) outrank the identifier the question is actually about,
- # which then never reaches _has_exact_symbol and the response claims the
- # symbol is unindexed. Score symbols on the extracted identifiers instead.
- symbol_query = query
canonical_symbol = _canonical_symbol_query(query)
- if canonical_symbol:
- symbol_query = canonical_symbol[1]
- if mode == "hybrid":
- _idents = _embedded_identifiers(query, names)
- # Without an identifier, score on the content words so stopwords such
- # as "on" can't match symbols like ``test_basic_auth_on_session``. A
- # query made only of stopwords keeps the raw text.
- symbol_query = (
- " ".join(_idents) if _idents else " ".join(content_terms(query)) or query
- )
-
- # Only the hybrid window collapses symbol rows, so only it over-fetches
- # them; path hits are file pages, already one row per file. Symbol mode
- # takes every candidate so ``fuzzy_omitted`` counts the names an exact hit hides.
- fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit
- if mode == "symbol":
- fetch = _MAX_CANDIDATES
+ plan = _SearchPlan(
+ query=query,
+ limit=limit,
+ page_type=page_type,
+ kind=kind,
+ symbol_kind=symbol_kind,
+ mode=mode,
+ names=names,
+ canonical_symbol=canonical_symbol,
+ symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),
+ fetch=_symbol_fetch(mode, limit),
+ multi=multi,
+ )
async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:
- s: list[dict] = []
- f: list[dict] = []
- paths: list[str] = []
- c: list[dict] = []
- if mode in ("symbol", "hybrid"):
- s = await search_symbols_single(
- ctx, symbol_query, fetch, symbol_kind=symbol_kind, kind=kind
- )
- if mode == "path":
- f = await search_paths_single(ctx, query, limit)
- async with get_session(ctx.session_factory) as session:
- paths = (await file_path_index(session, (await _get_repo(session)).id)).paths
- if mode == "hybrid":
- c = await _search_single_repo(ctx, query, limit, page_type, kind)
- for item in c:
- item["type"] = "page"
- await attach_hit_symbols(ctx, query, c)
- for rows in (s, f, c):
- _tag_repo(rows, ctx, multi)
- return s, f, paths, c
-
- answered = await _per_repo(repo, fed, _one_repo)
- contexts = [ctx for ctx, _ in answered]
- parts = [part for _, part in answered]
- # Symbol rows past each repo's ``limit``: they only backfill slots that
- # collapsing same-file rows frees, and never steer the exact-match signal.
- symbols = merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit)
- spare = merge_ranked((p[0][limit:] for p in parts), _neg_score("score"))
- files = merge_ranked((p[1] for p in parts), _neg_score("score"))
- indexed_paths = [path for p in parts for path in p[2]]
- # Re-ranked by relevance so a strong page in repo B isn't buried under
- # repo A's weak ones.
- concepts = merge_ranked((p[3] for p in parts), _neg_score("relevance_score"))
+ return await _search_one_repo(ctx, plan)
+
+ pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)
candidates = _identifier_candidates(query, mode, names)
- if mode == "symbol":
- symbols = _protect_exact_symbols(query, symbols)
- elif mode == "hybrid" and candidates:
- symbols = _protect_named_symbols(candidates, symbols)
- if mode == "path":
- files = _protect_exact_paths(query, files, indexed_paths)
-
- # Whether any returned symbol matches the query's identifier(s) exactly.
- # Computed once here so the hybrid interleave and the exact-match note below
- # agree on the same signal.
- exact = _has_exact_symbol(candidates, symbols) if candidates else False
- missing = _missing_named_symbols(candidates, symbols, bool(canonical_symbol), concepts)
- if missing:
- symbols = [s for s in symbols if _has_exact_symbol(candidates, [s])]
- spare = [s for s in spare if _has_exact_symbol(candidates, [s])]
+ _protect_exact(plan, pools, candidates)
+ exact, missing = _match_named(plan, pools, candidates)
- fuzzy_omitted = 0
- if mode == "symbol":
- # An exact hit is the answer; its fuzzy neighbours only cost bytes.
- # Summed over repos when federated, and a floor past the candidate cap.
- pool = symbols + spare
- named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []
- if named:
- ident = (canonical_symbol[1] if canonical_symbol else query).strip().lower()
- shown = {id(s) for s in named}
- fuzzy_omitted = sum(
- 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()
- )
- results = (named or symbols)[:limit]
- elif mode == "path":
- results = files[:limit]
- else: # hybrid: interleave symbol matches and concept pages for new files
- window = _hybrid_window(query, symbols, concepts, limit, exact, names)
- # One row per file, so ``limit`` buys distinct files (mode "symbol"
- # stays row-per-symbol: overloads there are the answer). Collapsing
- # keeps the window's order and frees the slots same-file rows took;
- # the next pages, then the next symbols, fill them.
- served = {id(item) for item in window}
- rest = [item for item in concepts + symbols + spare if id(item) not in served]
- results = _collapse_by_file(window + rest)[:limit]
- # A code-location query wants files to open, so a module, onboarding
- # or decision page is dropped from the window. Its slot is not
- # refilled: on the retrieval guard a refill bought +0.01 coverage at
- # limit 5 for -0.01 precision at limit 10. Kept when the caller asked
- # for pages by type or kind="doc".
- if not page_type and kind != "doc":
- results = [item for item in results if hit_file_path(item)]
-
- repository = None
- if not multi and contexts:
- async with get_session(contexts[0].session_factory) as session:
- repository = await _get_repo(session)
+ results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)
+ repository = await _single_repository(pools.contexts, multi)
response: dict = {
"results": results,
@@ -1195,59 +1344,16 @@ async def _structured_search(
}
if fuzzy_omitted:
response["fuzzy_omitted"] = fuzzy_omitted
- # Symbols first, then everything else the window holds: in symbol and
- # hybrid modes the ranked pool leads with symbol hits, and those are the
- # entries most likely to collapse onto one another (several symbols of one
- # file). Deduping them is the point.
- #
- # Bound to its own name, NOT to ``candidates``: that one holds the query's
- # identifiers, which the exact-match note below quotes. Rebinding it here
- # made the note quote file paths as if they were the identifiers asked for,
- # and — worse — made its gate true for any query with results at all, so a
- # prose query that names no identifier was told no symbol matched it.
+ # Not ``candidates``: that name holds the query's identifiers the note quotes.
if file_cands := file_candidates(results, limit=limit):
response["candidates"] = file_cands
- # Exact-match honesty: an identifier-shaped query whose target names no
- # indexed symbol still returns fuzzy neighbours. Say so, or the agent
- # anchors on a wrong hit that looks authoritative (their Alamofire
- # 44-overload read-spiral). Emit the boolean either way; a note only when
- # there is no exact hit to distinguish from the fuzz. ``candidates`` /
- # ``exact`` were computed above so ordering and this note stay consistent.
- if candidates:
- response["exact_match"] = exact and not missing
- if missing:
- shown = ", ".join(repr(c) for c in missing[:3])
- # An empty window has no page to qualify, and its grep_hint
- # already carries the sweep advice.
- pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if results else ""
- sweep = "" if (grep_hint and not results) else " " + EXHAUSTIVE_SWEEP_HINT
- response["note"] = (
- f"No indexed symbol is named {shown}, so no symbol is returned "
- f"for it.{pages} Recheck the spelling, or search a shorter part "
- f"of the name.{sweep}"
- )
- elif not exact:
- shown = ", ".join(repr(c) for c in candidates[:3])
- response["note"] = (
- f"No indexed symbol exactly matches {shown}. The results are "
- "fuzzy neighbours ranked by token overlap — confirm a hit names "
- "what you meant before relying on it. If you expected an exact "
- "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT
- )
- elif fuzzy_omitted:
- response["note"] = (
- f"{fuzzy_omitted} other symbols contain this name; search a "
- "longer or partial name to list them."
- )
- if grep_hint and not results:
- response["grep_hint"] = grep_hint
+ _attach_match_notes(response, candidates, exact, missing, grep_hint)
if fed is not None:
fed.annotate(response, results)
- # Last, so nothing above has to know the field is on its way out. Paths
- # first, so a page whose target_path is dropped keeps its page_id.
+ # Last: slims fields the steps above still read.
_slim_served_rows(
results,
- {(ctx.alias if multi else None): _page_language(ctx) for ctx in contexts},
+ {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},
)
_serve_snippets(results)
return response
diff --git a/tests/unit/server/mcp/test_embed_timeout.py b/tests/unit/server/mcp/test_embed_timeout.py
index 1bffb4832..0a6873b0c 100644
--- a/tests/unit/server/mcp/test_embed_timeout.py
+++ b/tests/unit/server/mcp/test_embed_timeout.py
@@ -81,7 +81,9 @@ def test_no_hardcoded_eight_second_embed_budget() -> None:
"""Pins the fix: the embed budget is resolved live, not baked in at import."""
from pathlib import Path
- text = Path(pipeline.__file__).read_text(encoding="utf-8")
+ text = "".join(
+ p.read_text(encoding="utf-8") for p in sorted(Path(pipeline.__file__).parent.glob("*.py"))
+ )
assert "_EMBED_TIMEOUT_S = 8.0" not in text
assert "_EMBED_TIMEOUT_S =" not in text # no hardcoded module-level constant at all
diff --git a/tests/unit/server/mcp/test_vector_search_timeout.py b/tests/unit/server/mcp/test_vector_search_timeout.py
index 512e01c51..c29650b2d 100644
--- a/tests/unit/server/mcp/test_vector_search_timeout.py
+++ b/tests/unit/server/mcp/test_vector_search_timeout.py
@@ -99,6 +99,6 @@ def test_no_hardcoded_eight_second_vector_budget() -> None:
from pathlib import Path
root = Path(tool_search.__file__).parent
- for name in ("tool_search.py", "_answer_pipeline.py"):
- text = (root / name).read_text(encoding="utf-8")
- assert "timeout=8.0" not in text, f"{name} still hardcodes the old vector budget"
+ for path in (root / "tool_search.py", *sorted((root / "_answer_pipeline").glob("*.py"))):
+ text = path.read_text(encoding="utf-8")
+ assert "timeout=8.0" not in text, f"{path.name} still hardcodes the old vector budget"