Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(mcp): split tool_search into a package of search stages (#4044)
Raghav Chamadiya7823cd1de1c1
diff --git a/docs/LINEAGE.md b/docs/LINEAGE.mdindex 47f571e0e..aa3271f58 100644--- a/docs/LINEAGE.md+++ b/docs/LINEAGE.md@@ -127,7 +127,7 @@ and infers the parameters and return values a behaviour-preserving signature nee| Year | Work | What we take from it | Where it lives ||---|---|---|---|| 2009 | Robertson and Zaragoza, *The probabilistic relevance framework: BM25 and beyond*, Foundations and Trends in IR | BM25 ranking over full-text search, with per-column weights | `persistence/search.py` |-| 2009 | Cormack, Clarke and Büttcher, *Reciprocal rank fusion outperforms Condorcet and individual rank learning methods*, SIGIR | Merging full-text and semantic results | `packages/server/src/repowise/server/mcp_server/tool_search.py` |+| 2009 | Cormack, Clarke and Büttcher, *Reciprocal rank fusion outperforms Condorcet and individual rank learning methods*, SIGIR | Merging full-text and semantic results | `packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py` |**How we checked it.** On 42 sealed retrieval tasks, held out from every round oftuning and graded without a model, `get_answer` found 0.876 of the files a fix needed,diff --git a/docs/reference/COMPUTED_GLOSSARY.md b/docs/reference/COMPUTED_GLOSSARY.mdindex 6fb15c7d1..9babb562f 100644--- a/docs/reference/COMPUTED_GLOSSARY.md+++ b/docs/reference/COMPUTED_GLOSSARY.md@@ -263,8 +263,8 @@ workspace overlays, MCP responses, and CLI output.| Answer payload | Cached `get_answer` result. | `get_answer()` | `{answer, citations, confidence, fallback_targets, retrieval}` || Retrieval hit | Search hit hydrated with page metadata and summary. | `tool_answer.py` retrieval pipeline | `{target_path: "src/auth.py", score: 3.2, summary: "..."}` || Retrieval dominance | Gating logic comparing top and second search scores. | `tool_answer.py` | Top score high enough to answer from dominant hit |-| Federated RRF score | Reciprocal rank fusion score for workspace search across repos. | `tool_search.py` | `rrf_score: 0.0164` |-| Confidence score | Normalized workspace search confidence. | `tool_search.py` | `confidence_score: 0.87` |+| Federated RRF score | Reciprocal rank fusion score for workspace search across repos. | `tool_search/retrieval.py` | `rrf_score: 0.0164` |+| Confidence score | Normalized workspace search confidence. | `tool_search/retrieval.py` | `confidence_score: 0.87` |## Persistence Tables And Stored Entitiesdiff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pydeleted file mode 100644index 791895736..000000000--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ /dev/null@@ -1,1558 +0,0 @@-"""MCP Tool 5: search_codebase — semantic search over the wiki."""--from __future__ import annotations--import asyncio-import contextlib-import logging-import re-from collections.abc import Container, Sequence-from dataclasses import dataclass-from typing import Any--from sqlalchemy import select--from repowise.core.generation.structural_labels import is_structural_title-from repowise.core.persistence.database import get_session-from repowise.core.persistence.models import (- GitMetadata,- Page,-)-from repowise.core.persistence.search import strip_leading_headings-from repowise.core.providers.embedding import store_has_semantic_vectors-from repowise.core.registry import ToolRecipe-from repowise.core.registry import mcp_tool_registry as mcp-from repowise.core.test_paths import is_test_path, is_test_related_path-from repowise.server.mcp_server._answer_pipeline import (- _FILENAME_LEG_RRF_K,- _RRF_K,- _RRF_SCORE_SCALE,- _SYMBOL_LEG_RRF_K,- _safe_filename_search,- _safe_symbol_search,- leg_ranks,-)-from repowise.server.mcp_server._budget import (- OmissionCollector,- register_post_enforce,- register_post_shed,-)-from repowise.server.mcp_server._federation import Federation, merge_ranked-from repowise.server.mcp_server._helpers import (- _VECTOR_TIMEOUT_ENV,- _get_exclude_spec,- _get_repo,- _resolve_repo_context,- attach_ignored_arguments,- filter_dicts_by_key,- resolve_enum_argument,- vector_search_timeout_s,-)-from repowise.server.mcp_server._hit_symbols import attach_hit_symbols-from repowise.server.mcp_server._line_hits import attach_line_hits-from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT, semantic_search_state-from repowise.server.mcp_server._meta import build_meta as _build_meta-from repowise.server.mcp_server._page_paths import (- FILE_ROW_TYPES,- PAGELESS_FILE,- add_row_paths,- file_candidates,- file_path_of,- hit_file_path,- pageless_path,-)-from repowise.server.mcp_server._query_shape import (- _DECISION_DOWNWEIGHT,- _MIN_RELEVANCE_SCORE,- _VALID_MODES,- NOT_THE_NAMED_SYMBOL,- _canonical_symbol_query,- _embedded_identifiers,- _fetch_limit_for,- _has_exact_symbol,- _identifier_candidates,- _is_why_shaped,- _looks_like_code_name,- _looks_like_exact_token,- _mark_not_the_named_symbol,- _names_a_path,- _qual_norm,- _resolve_mode,- _symbol_matches_name,- path_tokens,-)-from repowise.server.mcp_server._query_terms import content_terms-from repowise.server.mcp_server._references import path_identity, symbol_identity-from repowise.server.mcp_server._retrieval_rank import (- boost_named_paths,- rerank_pages_first,-)-from repowise.server.mcp_server.tool_search_symbols import (- _MAX_CANDIDATES,- IssueFiles,- file_path_index,- indexed_names,- issue_files,- search_paths_single,- search_symbols_single,-)--_log = logging.getLogger("repowise.mcp.search")--# Freshness tie-breaker, added (not multiplied) to a hit's fused relevance.-# The RRF-fused score spaces adjacent ranks ~0.05 apart, so this stays under-# one rank step: recency orders otherwise-comparable hits without overriding-# retrieval relevance. Recency itself scales it (0.5 for 90-day activity, 1.0-# for 30-day).-_FRESHNESS_TIEBREAK = 0.03---def _protect_exact_symbols(query: str, symbols: list[dict]) -> list[dict]:- """Stable-partition an exact identifier or ``path::Symbol`` hit to the head."""- if not symbols:- return symbols- canonical = _canonical_symbol_query(query)- qnorm = query.strip().lower().replace("\\", "/")-- def exact(item: dict) -> bool:- symbol_id = (item.get("symbol_id") or "").lower().replace("\\", "/")- if canonical:- return symbol_id == qnorm- return qnorm in {- (item.get("name") or "").lower(),- _qual_norm(item.get("qualified_name")),- }-- protected = [item for item in symbols if exact(item)]- return protected + [item for item in symbols if not exact(item)]---def _protect_named_symbols(candidates: list[str], symbols: list[dict]) -> list[dict]:- """Stable-partition symbols exactly named by identifiers embedded in prose."""- wanted = {candidate.strip().lower() for candidate in candidates if candidate.strip()}- wanted |= {_qual_norm(candidate) for candidate in candidates if candidate.strip()}-- protected = [item for item in symbols if _symbol_matches_name(item, wanted)]- return protected + [item for item in symbols if not _symbol_matches_name(item, wanted)]---def _protect_exact_paths(- query: str, files: list[dict], paths: Sequence[str] = ()-) -> list[dict]:- """Stable-partition an exact path hit ahead of basename/fuzzy neighbours."""- tokens = path_tokens(query, paths) or [query]- wanted = {t.strip().lower().replace("\\", "/") for t in tokens}- protected = [item for item in files if (item.get("file") or "").lower() in wanted]- return protected + [item for item in files if (item.get("file") or "").lower() not in wanted]---def _prose_dominates(query: str, identifiers: list[str]) -> bool:- """True when natural-language tokens outnumber the identifier tokens a query- carries: the query reads as prose that merely *mentions* a symbol, not a- symbol lookup dressed in a few words. Drives hybrid ordering below."""- ident_count = len(identifiers)- if ident_count == 0:- return False- total = len(re.findall(r"[A-Za-z0-9_]+", query))- return (total - ident_count) > ident_count---def _interleave_hybrid(- query: str,- symbols: list[dict],- concepts: list[dict],- limit: int,- exact: bool,- names: Container[str] | None = None,-) -> list[dict]:- """Order one hybrid result window from the two incomparable score scales.-- Symbol SQL scores (~43, +100 on an exact-name hit) and concept relevance- (0 to 1) can't be merge-sorted, so we interleave by block. Default: symbols- lead, reserving up to half the window for concept pages so a flood of symbol- matches can't truncate every page out.-- The exception is the score-scale trap. When NO returned symbol matches the- query's identifier exactly AND the query is mostly prose, leading with fuzzy- symbol hits buries the page the caller actually wants: the generic ``.get``- methods that outscore the ``answer.py`` page for "how does retrieval feed- synthesis in get_answer". There, concept pages lead and the fuzzy symbols- fall to the tail (nothing is dropped, only reordered within the window).- """- if not exact and concepts and _prose_dominates(query, _embedded_identifiers(query, names)):- reserved = min(len(symbols), limit // 2)- return (concepts[: max(1, limit - reserved)] + symbols)[:limit]- reserved = min(len(concepts), limit // 2)- return (symbols[: max(1, limit - reserved)] + concepts)[:limit]---# Code-location windows over-fetch this many times ``limit`` so collapsing-# same-file hits still leaves ``limit`` distinct files to serve.-_FILE_WINDOW_OVERFETCH = 2-# Other symbol names a collapsed row carries in ``symbols``.-_MERGED_SYMBOL_CAP = 5---def _collapse_by_file(hits: list[dict]) -> list[dict]:- """One row per (repo, file), best first; rows naming no file pass through.-- The first row of a file wins. A later symbol hit in the same file folds- into the winner's ``symbols`` as ``name:line`` (capped) instead of taking a- slot of its own, so ``limit`` buys distinct files; past the cap a trailing- ``+N more`` entry counts the rest. Precedent:- ``symbol_backed_pages`` collapses the same way for the concept tail.- """- first_of: dict[tuple, dict] = {}- extra: dict[int, int] = {}- out: list[dict] = []- for hit in hits:- path = hit_file_path(hit)- if path is None:- out.append(hit)- continue- key = (hit.get("repo"), path)- first = first_of.get(key)- if first is None:- first_of[key] = hit- out.append(hit)- elif hit.get("type") == "symbol" and first.get("type") == "symbol":- merged = first.setdefault("symbols", [])- if len(merged) < _MERGED_SYMBOL_CAP:- merged.append(f"{hit.get('name')}:{hit.get('start_line')}")- else:- extra[id(first)] = extra.get(id(first), 0) + 1- for hit in out:- if id(hit) in extra:- hit["symbols"].append(f"+{extra[id(hit)]} more")- return out---def _hybrid_window(- query: str,- symbols: list[dict],- concepts: list[dict],- limit: int,- exact: bool,- names: Container[str] | None = None,-) -> list[dict]:- """The hybrid window, with no page for a file a shown symbol row covers.-- Pages are dropped only for symbols the window actually shows, then the- window is rebuilt so the freed slots backfill. Concepts only shrink and the- symbol share only grows, so this settles in a few passes.- """- while True:- window = _interleave_hybrid(query, symbols, concepts, limit, exact, names)- shown = {item.get("file") for item in window if item.get("type") == "symbol"}- kept = [c for c in concepts if c.get("target_path") not in shown]- if len(kept) == len(concepts):- return window- concepts = kept---def _downweight_decisions(output: list[dict], query: str) -> None:- """Scale decision_record relevance in place unless the query is why-shaped."""- if _is_why_shaped(query):- return- for item in output:- if item.get("page_type") == "decision_record" and item.get("relevance_score"):- item["relevance_score"] = round(item["relevance_score"] * _DECISION_DOWNWEIGHT, 4)---# Test file pages compete against the implementation they exercise and win-# retrieval on any shared vocabulary ("conftest" for a fixtures question,-# "decision" for a downweight question) — observed live as a test file ranking-# #1 for a plain implementation query. A test is rarely a better first Read than-# the code under test, so demote it, unless the query is explicitly about tests.-_TEST_DOWNWEIGHT = 0.6--_TEST_QUERY_RE = re.compile(- r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",- re.IGNORECASE,-)---def _is_test_query(query: str) -> bool:- """True when the query is explicitly about tests, so test pages rank naturally."""- return bool(_TEST_QUERY_RE.search(query))---def _is_test_page(item: dict) -> bool:- """True when a hit is a whole-file row for a test file.-- Tests only, deliberately not test support: a ``conftest.py`` or a fixture- module is often exactly what the person searching wanted, so it keeps its- rank. The ``kind`` filter counts both (see :func:`_classify_hit_kind`) —- asking for tests and asking to rank tests lower are different questions.- """- return item.get("page_type") in FILE_ROW_TYPES and is_test_path(item.get("target_path") or "")---def _downweight_test_pages(output: list[dict], query: str) -> None:- """Scale test file_page relevance in place unless the query is about tests."""- if _is_test_query(query):- return- for item in output:- if item.get("relevance_score") and _is_test_page(item):- item["relevance_score"] = round(item["relevance_score"] * _TEST_DOWNWEIGHT, 4)---def _sort_demoting_noise(output: list[dict], query: str) -> None:- """Sort by relevance, ranking retrieval noise below every real page.-- Two classes crowd the top ranks on a plain implementation query and are- demoted absolutely for their query class (the relevance score still orders- each class among itself):-- - decision records — short dense titles win cosine similarity by a margin- wider than the multiplicative down-weight (observed live as 5/5 irrelevant- decisions for one query). Rationale questions are get_why's territory, so- a why-shaped query ranks them naturally instead.- - test file pages — a test is rarely a better first Read than the code it- exercises. A query explicitly about tests ranks them naturally instead.- """- why = _is_why_shaped(query)- test_focused = _is_test_query(query)-- def key(item: dict) -> tuple:- pt = item.get("page_type")- is_decision = (not why) and pt == "decision_record"- is_test = (not test_focused) and _is_test_page(item)- return (1 if (is_decision or is_test) else 0, -(item.get("relevance_score") or 0.0))-- output.sort(key=key)---def _norm_decision_title(title: str) -> str:- """Normalize a decision title to collapse near-duplicate phrasings."""- return re.sub(r"[^a-z0-9]+", " ", (title or "").lower()).strip()---def _dedup_decisions(output: list[dict]) -> list[dict]:- """Collapse near-duplicate decision records by normalized title (order-preserving).-- Live search returned five near-identical "CLI incremental update regenerates- only affected pages" variants filling positions 3-8. Callers dedup AFTER the- score sort, so the surviving variant is the highest-scored one; non-decision- pages always pass through untouched.- """- seen: set[str] = set()- out: list[dict] = []- for item in output:- if item.get("page_type") == "decision_record":- key = _norm_decision_title(item.get("title", ""))- if key and key in seen:- continue- seen.add(key)- out.append(item)- return out---async def _non_decision_fallback(ctx, query: str, fetch_limit: int) -> list[dict]:- """Wider re-fetch keeping only non-decision pages.-- Guard for the window-saturation failure: when the entire over-fetched- window is decision records, demotion has nothing to promote and a- non-why query returns zero implementation pages. Fetch 4x wider, drop- decisions, and let the caller merge the survivors.- """- results = []- src = "vector"- # Skipped on a keyless index, which then falls through to the FTS branch.- if store_has_semantic_vectors(ctx.vector_store):- try:- results = await asyncio.wait_for(- ctx.vector_store.search(query, limit=fetch_limit * 4),- timeout=vector_search_timeout_s(),- )- except TimeoutError:- _log.warning("Vector re-fetch timed out; falling back to full-text")- except Exception:- _log.debug("Vector re-fetch failed; falling back to full-text", exc_info=True)- if not results:- src = "fts"- with contextlib.suppress(Exception):- results = await ctx.fts.search(query, limit=fetch_limit * 4)-- out = []- for r in results:- if r.page_type == "decision_record":- continue- if r.score < _MIN_RELEVANCE_SCORE:- continue- out.append(- {- "page_id": r.page_id,- "title": r.title,- "page_type": r.page_type,- "snippet": r.snippet,- "relevance_score": r.score,- "sources": [src],- }- )- return out---async def _rescue_all_decision_window(- ctx, output: list[dict], query: str, fetch_limit: int-) -> list[dict]:- """Merge non-decision pages into an all-decision result window.-- No-op unless the query is non-why AND every current hit is a decision- record. Deduplicates by page_id.- """- if not output or _is_why_shaped(query):- return output- if not all(item.get("page_type") == "decision_record" for item in output):- return output- fallback = await _non_decision_fallback(ctx, query, fetch_limit)- seen = {item["page_id"] for item in output}- output.extend(item for item in fallback if item["page_id"] not in seen)- return output---# Slots at the tail of a concept window reserved for files reached through the-# symbol index rather than through their page. A generated file page renders-# only public symbols, so a question about a private helper or a local name has-# nothing to match in either the full-text index or the embedding. The symbol-# index is where those names live. One slot in three, taken from the weakest-# end of the window, is a cheap price for reaching a class of file the page-# retrievers structurally cannot see. No-op when the symbol leg finds nothing.-_SYMBOL_TAIL_DIVISOR = 3---def _symbol_tail_reserve(limit: int) -> int:- """How many tail slots the symbol leg may take from a window of *limit*."""- return limit // _SYMBOL_TAIL_DIVISOR---def _reserve_symbol_tail(output: list[dict], limit: int, kind: str | None) -> list[dict]:- """Give the weakest window slots to symbol-leg files ranked past ``limit``.-- Concept hits keep the head. Files the symbol leg reached that fusion left- outside the window take up to :func:`_symbol_tail_reserve` slots, and the- concept hits they displaced fall in behind them (nothing is dropped).- """- reserve = _symbol_tail_reserve(limit)- if reserve <= 0 or kind in ("config", "doc"):- # config/doc windows are asking for pages with no symbols in them.- return output- extra = [item for item in output[limit:] if "symbol" in item.get("sources", ())][:reserve]- if not extra:- return output- head = output[: limit - len(extra)]- taken = {id(item) for item in head + extra}- return head + extra + [item for item in output if id(item) not in taken]---# Path-prefix heuristics for the ``kind`` filter. We classify a hit's-# target_path against these prefixes; if none match, the hit falls into-# ``other`` and is dropped only when the caller asked for a specific kind.-# Tests are not in this list: they come from ``repowise.core.test_paths``, the-# same rules that stamped the file's ``is_test`` flag at ingestion.-_CONFIG_PATH_TOKENS = (- "pyproject.toml",- "package.json",- "tsconfig",- "setup.py",- "setup.cfg",- "/.github/",- "dockerfile",- ".yml",- ".yaml",- ".toml",- ".ini",- ".cfg",- "lockfile",- "package-lock",- "uv.lock",- "poetry.lock",-)---def _classify_hit_kind(target_path: str, page_type: str) -> str:- """Bucket a hit into implementation / test / config / doc.-- Pages without a file behind them (decision records, repo overviews,- onboarding pages — all have empty ``target_path``) are docs: letting- them fall through to the path heuristics classified every decision- record as "implementation", so ``kind="implementation"`` returned- decision pages instead of filtering them out.-- ``test`` is the union of tests and test support here: someone asking for- ``kind="implementation"`` does not want a ``conftest.py`` back, and someone- asking for ``kind="test"`` does.-- Config wins over test, which the token list this replaced never had to- decide: ``.github/workflows/tests.yml`` is a workflow whatever it is named,- and dropping it from ``kind="config"`` would be the more surprising answer.- """- tp = (target_path or "").lower()- if page_type in ("module_page", "symbol_spotlight") or tp.endswith(".md"):- return "doc"- if not tp or page_type not in FILE_ROW_TYPES:- return "doc"- if any(tok in tp for tok in _CONFIG_PATH_TOKENS):- return "config"- # Original case, not ``tp``: the camel (FooTest.java) and .NET project-dir- # (Foo.Tests/) rules are deliberately case-sensitive.- if is_test_related_path(target_path):- return "test"- return "implementation"---# Every value :func:`_classify_hit_kind` can return; anything else can only ever-# match nothing, which is why an unknown kind is dropped rather than filtered on.-_VALID_KINDS = frozenset({"implementation", "test", "config", "doc"})---def _filter_by_kind(output: list[dict], kind: str | None) -> list[dict]:- """Keep only hits classified as ``kind`` (no-op when ``kind`` is falsy).-- Runs on the over-fetched list BEFORE the limit cut so the caller still- gets up to ``limit`` results of the requested kind.- """- if not kind:- return output- return [- item- for item in output- if _classify_hit_kind(item.get("target_path", ""), item.get("page_type", "")) == kind- ]---def _attach_paths(output: list[dict], page_info: dict) -> None:- """Stamp canonical file paths and split symbol ids into their own field.-- A ``symbol_spotlight``'s target_path is ``file.py::Symbol``: a page id, and- not a file path. Preserve it as ``symbol_id`` for ``get_symbol`` and expose- only the file portion through ``target_path`` / ``file``.-- This lives in one function because it did not used to. The concept branch- of ``search_codebase`` set ``file``; ``_search_single_repo``, which is what- the hybrid and federated branches call, did not. A query carrying a- CamelCase token routes to hybrid, so the tool's headline path kept serving- page ids in a field read as a path. Measured on the dev-fix1 predictions:- 45.4% of the python served paths still carried ``::``, and resolving them- takes gold containment from 19 to 39 of 50 instances.- """- for item in output:- item["target_path"] = page_info.get(item["page_id"], item.get("target_path", ""))- if "::" in item["target_path"]:- item["symbol_id"] = symbol_identity(item["target_path"])- item["target_path"] = path_identity(item["target_path"].split("::", 1)[0])- item["file"] = item["target_path"]---def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:- """Strip ``page_id`` from every result that can rebuild it, in place.-- A page id is ``compute_page_id(page_type, target_path)`` — literally- ``f"{page_type}:{target_path}"`` (``core/generation/models.py``) — and both- halves ship in the same row. So the field is 229-369 characters per- response, 7.8-10.3% of the payload, restating two of its own siblings. It- is ranking plumbing: every internal use above keys the fused dict on it,- and none of that needs to reach the wire.-- Conditional rather than unconditional, and the condition is the derivation- itself. ``_attach_paths`` writes ``target_path: ""`` for a hit whose Page- row did not load, and a row like that cannot be rebuilt — so it keeps its- id instead of losing one. The rule is "drop it only where it is provably- redundant", which is also the check: if this ever stops firing, the- invariant it rests on has changed.-- Consumers rebuild with the same expression; ``ui/src/chat/source-citations``- does exactly that, and used to skip any row whose ``page_id`` was missing.-- ``add_row_paths`` runs first and deletes ``target_path`` wherever it equals- the new ``path``, so the id's second half is ``target_path`` when present- and ``path`` otherwise. Without that fallback an ordinary file row rebuilt- as ``"file_page:"`` and kept its id.- """- for item in results:- # A symbol page's id ends in ``::Symbol`` and ``_attach_paths`` moved that- # half into ``symbol_id``, so ``target_path`` is only its file.- target = (- item.get("symbol_id")- if item.get("page_type") == "symbol_spotlight"- else None- ) or item.get("target_path") or item.get("path", "")- derived = f"{item.get('page_type', '')}:{target}"- # A row for a file with no page has an id that names no page.- if item.get("page_id") == derived or item.get("page_type") == PAGELESS_FILE:- item.pop("page_id", None)- return results---def _page_language(ctx) -> str:- """The language *ctx*'s structural page titles were generated in."""- try:- from repowise.core.repo_config import load_repo_config-- return str(load_repo_config(ctx.path).get("language") or "en")- except Exception:- return "en"---def _slim_served_rows(- rows: list[dict], languages: dict[str | None, str] | None = None-) -> list[dict]:- """The last step before a search reply leaves: one location per row.-- ``path`` (plus ``symbol_id`` on symbol rows) is the row's only location, so- ``file``, a ``page_id`` that rebuilds from them, and a ``title`` that is just- the structural label wrapped around them are all dropped. Each is dropped- only where the caller can provably rebuild it. ``sources: ["fts"]`` goes when- the reply's own ``_meta.semantic_search`` already says retrieval is- full-text-only; an index with embeddings is unchanged.-- *languages* maps a row's ``repo`` alias (``None`` for a single-repo reply) to- the language its titles were generated in.- """- add_row_paths(rows)- _drop_derivable_page_ids(rows)- fts_only = semantic_search_state() is False- for row in rows:- target = row.get("symbol_id") or row.get("path")- if target and is_structural_title(- (languages or {}).get(row.get("repo"), "en"),- row.get("page_type", ""),- target,- row.get("title"),- ):- row.pop("title", None)- if fts_only and row.get("sources") == ["fts"]:- row.pop("sources", None)- return rows---def _serve_snippets(results: list[dict]) -> None:- """Drop each snippet's leading page heading, after ranking has read it."""- for item in results:- if item.get("snippet"):- item["snippet"] = strip_leading_headings(item["snippet"])---def _drop_internal_ranking_fields(results: list[dict]) -> None:- """Keep calibration diagnostics internal to the ranking pipeline."""- for item in results:- item.pop("_coverage", None)- item.pop("_coverage_multiplier", None)- item.pop("_confidence_score_factor", None)- item.pop("_raw_score", None)---async def _load_page_info(- session, output: list[dict], *, with_git: bool = False-) -> tuple[dict, set, dict]:- """Batch-load target paths, tombstones, and git.-- Returns ``(page_info, tombstoned, git_map)`` where ``page_info`` maps- page_id -> target_path, ``tombstoned`` is the set of tombstoned page_ids,- and ``git_map`` maps file_path -> GitMetadata (empty unless ``with_git``).- """- page_ids = [item["page_id"] for item in output]- res = await session.execute(- select(Page.id, Page.target_path, Page.freshness_status).where(Page.id.in_(page_ids))- )- rows = res.all()- page_info = {row[0]: row[1] for row in rows}- tombstoned = {row[0] for row in rows if row[2] == "tombstone"}-- git_map: dict[str, GitMetadata] = {}- if with_git:- target_paths = [tp for tp in page_info.values() if tp]- if target_paths:- git_res = await session.execute(- select(GitMetadata).where(GitMetadata.file_path.in_(target_paths))- )- git_map = {g.file_path: g for g in git_res.scalars().all()}- return page_info, tombstoned, git_map---def _apply_freshness_boost(item: dict, gm: GitMetadata | None) -> None:- """Nudge a recently-active file up as a bounded tie-breaker (in place).-- Additive, not multiplicative: the fused relevance score is RRF-based, which- compresses adjacent ranks to roughly ``_FRESHNESS_TIEBREAK`` apart. A- percentage boost on that scale would let a fresh rank-2 hit leapfrog a- stale rank-0 hit, overriding retrieval relevance with recency. Sized below- one rank step, freshness only orders hits retrieval already ranks close- together.- """- if not gm or not item.get("relevance_score"):- return- c30 = gm.commit_count_30d or 0- c90 = gm.commit_count_90d or 0- if c30 > 0:- recency = 1.0- elif c90 > 0:- recency = 0.5- else:- recency = 0.0- item["relevance_score"] = round(item["relevance_score"] + _FRESHNESS_TIEBREAK * recency, 4)---def _assign_confidence(output: list[dict], score_key: str, target_key: str) -> None:- """Derive ``target_key`` for each item from its ``score_key`` position (in place)."""- if not output:- return- max_score = max((item.get(score_key) or 0) for item in output)- for item in output:- raw = item.get(score_key) or 0- item[target_key] = round(raw / max_score, 2) if max_score > 0 else 0.0---async def _wait_for_vector_store(ctx) -> None:- """Block (bounded) until the vector store signals readiness, if it tracks it."""- if ctx.vector_store_ready is not None:- with contextlib.suppress(TimeoutError):- await asyncio.wait_for(ctx.vector_store_ready.wait(), timeout=30.0)---async def _safe_fts(ctx, query: str, limit: int) -> list:- """FTS search, bounded and failure-swallowing (returns [] on error/timeout)."""- if ctx.fts is None:- return []- with contextlib.suppress(Exception):- return await asyncio.wait_for(ctx.fts.search(query, limit=limit), timeout=5.0)- return []---async def _safe_vector(ctx, query: str, limit: int) -> list:- """Vector search, bounded and failure-swallowing (returns [] on error/timeout).-- Readiness is awaited by the caller (``_wait_for_vector_store``) before the- fused retrieve runs, so this path does not re-wait.-- Returns nothing on a keyless index. ``store_has_semantic_vectors`` explains- why the mock's vectors cannot be ranked on; the reason this is enforced here- rather than in ``_fused_retrieve`` is that this function is the only place- the vector leg is entered, so a later caller inherits the guard.- """- if ctx.vector_store is None or not store_has_semantic_vectors(ctx.vector_store):- return []- try:- return await asyncio.wait_for(- ctx.vector_store.search(query, limit=limit), timeout=vector_search_timeout_s()- )- except TimeoutError:- # Not suppressed silently: a timeout here drops the whole semantic leg- # and the caller returns full-text-only hits that look like a normal- # result set, which is how #1678 read as "semantic search finds nothing".- _log.warning(- "Vector search exceeded its %gs budget; returning no semantic hits. "- "Raise it with %s=<seconds>.",- vector_search_timeout_s(),- _VECTOR_TIMEOUT_ENV,- )- except Exception:- _log.debug("Vector search failed; returning no semantic hits", exc_info=True)- return []---def _symbol_leg_target(fused: dict[str, dict], page_id: str) -> str:- """The fused page a symbol-leg file page credits: the best page of its file.-- The leg names files, while the page legs may have reached the same file- through a symbol page. Crediting only the file page id would leave that- hit unlifted and add a weak twin that collapsing then drops.- """- path = page_id.partition(":")[2]- best = None- for pid, entry in fused.items():- if file_path_of(entry["page_type"], pid.partition(":")[2]) != path:- continue- if best is None or entry["_rrf"] > fused[best]["_rrf"]:- best = pid- return best or page_id---def _fused_entry(r) -> dict:- """Seed a fused-result dict from a retriever hit (RRF score added by caller)."""- entry = {- "page_id": r.page_id,- "title": r.title,- "page_type": r.page_type,- "snippet": r.snippet,- "_rrf": 0.0,- "_sources": set(),- }- if r.page_type == PAGELESS_FILE:- # No page row to load it from later.- entry["target_path"] = pageless_path(r.page_id)- return entry---async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | None) -> list[dict]:- """Retrieve by fusing FTS, vector and symbol search via Reciprocal Rank Fusion.-- The retrievers run in parallel; a hit's score is the sum of- ``1/(rank + k)`` over the retrievers that surfaced it, scaled into the- BM25 range the downstream relevance gates were tuned against. Each hit- carries a ``sources`` list (``"fts"``, ``"vector"``, ``"symbol"``,- ``"filename"``): a page found by several retrievers is a stronger match- than one found by one mode. The symbol and filename legs also name files- that have no page, as ``page_type: "file"`` rows.-- This mirrors get_answer's ``hybrid_retrieve`` so the two entry points rank- the same corpus identically. The prior path ran vector search and fell- back to FTS only when vector returned nothing, so a page FTS ranked highly- but vector missed never surfaced here.-- A hit is admitted only when its raw retriever score clears- ``_MIN_RELEVANCE_SCORE`` — the fused score is rank-based and would- otherwise sit well above the floor for every window hit, letting an- off-topic query return its nearest-but-unrelated neighbours. Rank position- (for RRF) is the hit's place in the retriever's own list, unchanged by the- floor.- """- fts_results, vec_results, sym_results, name_results = await asyncio.gather(- _safe_fts(ctx, query, fetch_limit),- _safe_vector(ctx, query, fetch_limit),- _safe_symbol_search(ctx, query, pageless=True),- _safe_filename_search(ctx, query),- )-- fused: dict[str, dict] = {}- for rank, r in enumerate(vec_results):- if r.score < _MIN_RELEVANCE_SCORE:- continue- entry = fused.setdefault(r.page_id, _fused_entry(r))- entry["_rrf"] += 1.0 / (rank + _RRF_K)- entry["_sources"].add("vector")- for rank, r in enumerate(fts_results):- if r.score < _MIN_RELEVANCE_SCORE:- continue- entry = fused.setdefault(r.page_id, _fused_entry(r))- entry["_rrf"] += 1.0 / (rank + _RRF_K)- entry["_sources"].add("fts")- # Files whose symbol names the query's words spell, paged or not. Weighted- # below the page legs, as in get_answer; it has no raw score to floor.- for rank, r in leg_ranks(sym_results):- entry = fused.setdefault(_symbol_leg_target(fused, r.page_id), _fused_entry(r))- entry["_rrf"] += 1.0 / (rank + _SYMBOL_LEG_RRF_K)- entry["_sources"].add("symbol")- # Files whose file name the query's words spell, weighted as a name match.- # It adds files the other legs missed and never reorders a page they found.- # A name of three or more words the query spells in full weighs like a page hit.- for rank, (r, full) in leg_ranks(name_results):- target = _symbol_leg_target(fused, r.page_id)- if target in fused and r.page_type != PAGELESS_FILE:- continue- entry = fused.setdefault(target, _fused_entry(r))- entry["_rrf"] += 1.0 / (rank + (_RRF_K if full else _FILENAME_LEG_RRF_K))- entry["_sources"].add("filename")-- output: list[dict] = []- for entry in fused.values():- if page_type and entry["page_type"] != page_type:- continue- entry["relevance_score"] = round(entry.pop("_rrf") * _RRF_SCORE_SCALE, 4)- entry["sources"] = sorted(entry.pop("_sources"))- output.append(entry)- output.sort(key=lambda item: item["relevance_score"], reverse=True)- return output---async def _search_single_repo(- ctx, query: str, limit: int, page_type: str | None, kind: str | None = None-) -> list[dict]:- """Run search against a single repo context.-- Fuses FTS and vector retrieval via RRF (see ``_fused_retrieve``), so each- hit carries a ``sources`` list naming the retrievers that surfaced it- rather than a single backend label.-- The ``kind`` filter runs here, on the over-fetched list and BEFORE the- limit cut — filtering after per-repo truncation returned fewer than- ``limit`` results (frequently zero) in the federated path.- """- await _wait_for_vector_store(ctx)-- fetch_limit = _fetch_limit_for(limit, kind)- output = await _fused_retrieve(ctx, query, fetch_limit, page_type)-- _downweight_decisions(output, query)- output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)-- # Attach target_path and drop excluded hits per repo (so federated search- # honours each repo's own exclude_patterns) before ranking. Runs on the- # over-fetched list so the kind filter below has real headroom. Load- # precedes the sort/demotion so they see the page metadata they key on- # (test demotion needs target_path to classify a test file page).- if output:- async with get_session(ctx.session_factory) as session:- page_info, tombstoned, _ = await _load_page_info(session, output)- path_index = await file_path_index(session, (await _get_repo(session)).id)- output = [item for item in output if item["page_id"] not in tombstoned]- _attach_paths(output, page_info)- output = filter_dicts_by_key(output, "target_path", _get_exclude_spec(ctx.path))- boost_named_paths(- output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"- )-- _downweight_test_pages(output, query)- _sort_demoting_noise(output, query)- output = _dedup_decisions(output)-- output = _filter_by_kind(output, kind)- return _collapse_by_file(output)[:limit]---def _neg_score(key: str):- """Best-first sort key on a row's *key* score, for :func:`merge_ranked`."""- return lambda item: -(item.get(key) or 0.0)---async def _federated_search(- query: str, limit: int, page_type: str | None, kind: str | None, fed: Federation-) -> dict:- """Search across all repos using Reciprocal Rank Fusion."""-- async def _one_repo(ctx) -> list[dict]:- repo_results = await _search_single_repo(ctx, query, limit, page_type, kind)- for rank, item in enumerate(repo_results):- item["repo"] = ctx.alias- item["rrf_score"] = 1.0 / (rank + 60) # RRF constant k=60- return repo_results-- answered = await fed.run(_one_repo)- contexts = [ctx for ctx, _ in answered]- all_results = merge_ranked(- (rows for _, rows in answered), _neg_score("rrf_score"), per_repo_cap=limit- )- output = all_results[:limit]-- # Derive confidence from RRF position- _assign_confidence(output, "rrf_score", "confidence_score")-- # No single indexed commit to compare a live HEAD against: per-repo- # freshness rides on ``repos_stale`` and the rows' ``stale`` flag instead.- response: dict = {"results": output, "_meta": _build_meta()}- fed.annotate(response, output)- # Drawn from every repo's ranked list, not the merged cut: a workspace- # search that spends its window on module pages should still be able to- # name files.- if candidates := file_candidates(all_results, limit=limit):- response["candidates"] = candidates- # Last, so nothing above has to know the field is on its way out.- _slim_served_rows(output, {ctx.alias: _page_language(ctx) for ctx in contexts})- _serve_snippets(output)- return response---def _refresh_served_targets(response: dict, _collector: OmissionCollector | None) -> None:- """Re-derive target-scoped freshness from the hits that survived shedding.-- ``_meta.targets`` claims which files the response served. Shedding decides- that, so this has to run after it or the claim names rows the caller never- received.- """- if response.get("truncated") and isinstance(response.get("_meta"), dict):- response["_meta"]["targets"] = _result_paths(response.get("results") or [])---def _refresh_served_targets_final(response: dict) -> None:- """Same claim, re-derived after the final size guard may have cut again."""- _refresh_served_targets(response, None)---register_post_shed("search_codebase", _refresh_served_targets)-register_post_enforce("search_codebase", _refresh_served_targets_final)---def _result_paths(results: list[dict]) -> list[str]:- """File paths a result set serves, for target-scoped freshness.-- Symbol hits carry ``file``; page hits are resolved through their page type,- so a module page's group key and an onboarding slot resolve to nothing- rather than being handed on as if they were files. Freshness is per-file- git metadata, and a directory has none, so a page id reaching here could- only ever fail to match.-- An empty result set returns ``[]`` (meaning "no file content served",- which suppresses the repo-level stale warning by design).- """- paths: list[str] = []- for item in results:- p = hit_file_path(item)- if p:- paths.append(p)- return paths---def _grep_hint_for(query: str, names: Container[str] | None = None) -> str | None:- """Zero-result recovery hint for identifier-shaped queries, else ``None``.-- Only attached when the search produced nothing (see call sites) — a- populated result set needs no escape hatch. Points back into the tool- surface first; Grep is named only for the one job it genuinely wins:- an exhaustive literal sweep of every usage (e.g. before a rename).- """- if _looks_like_exact_token(query):- return (- f"No indexed match for identifier {query!r}. Retry with "- 'mode="symbol" (or check spelling/casing). ' + EXHAUSTIVE_SWEEP_HINT- )- if idents := _embedded_identifiers(query, names):- shown = ", ".join(repr(t) for t in idents[:3])- return (- f"Query names identifier(s) {shown} but nothing matched. Search "- 'the identifier alone with mode="symbol", then pipe the hit '- "into get_symbol for its body. " + EXHAUSTIVE_SWEEP_HINT- )- return None---async def _per_repo(repo: str | None, fed: Federation | None, fn) -> list[tuple[Any, Any]]:- """``(context, fn(context))`` for the one repo asked for, or every repo via *fed*."""- if fed is not None:- return await fed.run(fn)- ctx = await _resolve_repo_context(repo)- return [(ctx, await fn(ctx))]---def _lead_candidates(response: dict, issue: IssueFiles | None, limit: int) -> None:- """Trace files lead ``candidates``. A named identifier's defining file only- fills a free slot after the ranked ones: ranked above them, it displaces- better hits on ordinary questions. Capped at ``limit``."""- if issue is None or not (issue.traced or issue.named):- return- ranked = [c["path"] for c in response.get("candidates") or []]- paths = list(dict.fromkeys(issue.traced + ranked + issue.named))- response["candidates"] = [{"path": p} for p in paths[:limit]]---def _tag_repo(items: list[dict], ctx, multi: bool) -> None:- if multi:- for item in items:- item["repo"] = ctx.alias---def _missing_named_symbols(- candidates: list[str], symbols: list[dict], canonical: bool, concepts: list[dict]-) -> list[str]:- """Code-shaped names the query asks after that no returned symbol matches- exactly (or returned module path carries): they do not exist here. Judged- per name, so an indexed name beside a missing one hides nothing. Their- fuzzy neighbours would stand in for them, so the caller keeps only exact- symbols; the pages are marked here as related to the question, not the- symbol."""- if canonical:- return []- page_paths = [c.get("target_path") or "" for c in concepts]- missing = [- c- for c in candidates- if _looks_like_code_name(c)- and not _has_exact_symbol([c], symbols)- and not _names_a_path(c, page_paths)- ]- if missing:- _mark_not_the_named_symbol(concepts)- return missing---@dataclass(frozen=True)-class _SearchPlan:- """One structured search: the caller's arguments plus the derived query."""-- query: str- limit: int- page_type: str | None- kind: str | None- symbol_kind: str | None- mode: str- names: Container[str] | None- canonical_symbol: tuple[Any, str] | None- symbol_query: str- fetch: int- multi: bool---@dataclass-class _Pools:- """Each row kind merged and ranked across the repos searched."""-- contexts: list[Any]- symbols: list[dict]- spare: list[dict]- files: list[dict]- indexed_paths: list[str]- concepts: list[dict]---def _symbol_query_for(- query: str, mode: str, names: Container[str] | None, canonical_symbol: Any-) -> str:- """The text the symbol scorer ranks on."""- # Scored on the identifiers, not the prose: words like "is" would match- # ``is_ci`` and bury the symbol the question names.- symbol_query = query- if canonical_symbol:- symbol_query = canonical_symbol[1]- if mode == "hybrid":- _idents = _embedded_identifiers(query, names)- # No identifier: content words, else the raw text.- symbol_query = (- " ".join(_idents) if _idents else " ".join(content_terms(query)) or query- )- return symbol_query---def _symbol_fetch(mode: str, limit: int) -> int:- """How many symbol rows each repo fetches."""- # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every- # candidate so ``fuzzy_omitted`` can count what an exact hit hides.- fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit- if mode == "symbol":- fetch = _MAX_CANDIDATES- return fetch---async def _search_one_repo(- ctx: Any, plan: _SearchPlan-) -> tuple[list[dict], list[dict], list[str], list[dict]]:- """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""- s: list[dict] = []- f: list[dict] = []- paths: list[str] = []- c: list[dict] = []- if plan.mode in ("symbol", "hybrid"):- s = await search_symbols_single(- ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind- )- if plan.mode == "path":- f = await search_paths_single(ctx, plan.query, plan.limit)- async with get_session(ctx.session_factory) as session:- paths = (await file_path_index(session, (await _get_repo(session)).id)).paths- if plan.mode == "hybrid":- c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)- for item in c:- item["type"] = "page"- await attach_hit_symbols(ctx, plan.query, c)- for rows in (s, f, c):- _tag_repo(rows, ctx, plan.multi)- return s, f, paths, c---def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:- """Merge each repo's rows into one ranked pool per row kind."""- parts = [part for _, part in answered]- return _Pools(- contexts=[ctx for ctx, _ in answered],- symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),- # Rows past each repo's limit only backfill slots that collapsing frees.- spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),- files=merge_ranked((p[1] for p in parts), _neg_score("score")),- indexed_paths=[path for p in parts for path in p[2]],- concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),- )---def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:- """Keep exact name and path matches ahead of their fuzzy neighbours."""- if plan.mode == "symbol":- pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)- elif plan.mode == "hybrid" and candidates:- pools.symbols = _protect_named_symbols(candidates, pools.symbols)- if plan.mode == "path":- pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)---def _symbol_mode_results(- plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool-) -> tuple[list[dict], int]:- """Symbol mode rows, and how many fuzzy names an exact hit hides."""- # An exact hit is the answer; its fuzzy neighbours are only counted.- fuzzy_omitted = 0- pool = pools.symbols + pools.spare- named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []- if named:- canonical_symbol = plan.canonical_symbol- ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()- shown = {id(s) for s in named}- fuzzy_omitted = sum(- 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()- )- return (named or pools.symbols)[: plan.limit], fuzzy_omitted---def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:- """Hybrid rows: symbol matches interleaved with concept pages for new files."""- window = _hybrid_window(- plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names- )- # One row per file; freed slots fill with the next pages, then symbols.- served = {id(item) for item in window}- rest = [- item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served- ]- results = _collapse_by_file(window + rest)[: plan.limit]- # Non-file pages leave the window unrefilled unless the caller asked for- # pages by type or kind="doc".- if not plan.page_type and plan.kind != "doc":- results = [item for item in results if hit_file_path(item)]- return results---def _select_results(- plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool-) -> tuple[list[dict], int]:- """The served rows for the plan's mode, and the symbol-mode fuzzy count."""- if plan.mode == "symbol":- return _symbol_mode_results(plan, pools, candidates, exact)- if plan.mode == "path":- return pools.files[: plan.limit], 0- return _hybrid_mode_results(plan, pools, exact), 0---async def _single_repository(contexts: list[Any], multi: bool) -> Any:- """The repository row a single-repo search ran against, else None."""- if multi or not contexts:- return None- async with get_session(contexts[0].session_factory) as session:- return await _get_repo(session)---def _exact_match_note(- candidates: list[str],- exact: bool,- missing: list[str],- fuzzy_omitted: int,- has_results: bool,- grep_hint: str | None,-) -> str | None:- """The note qualifying an identifier query's results, if one is owed."""- if missing:- shown = ", ".join(repr(c) for c in missing[:3])- # An empty window's grep_hint already carries the sweep advice.- pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""- sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT- return (- f"No indexed symbol is named {shown}, so no symbol is returned "- f"for it.{pages} Recheck the spelling, or search a shorter part "- f"of the name.{sweep}"- )- if not exact:- shown = ", ".join(repr(c) for c in candidates[:3])- return (- f"No indexed symbol exactly matches {shown}. The results are "- "fuzzy neighbours ranked by token overlap — confirm a hit names "- "what you meant before relying on it. If you expected an exact "- "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT- )- if fuzzy_omitted:- return (- f"{fuzzy_omitted} other symbols contain this name; search a "- "longer or partial name to list them."- )- return None---def _match_named(- plan: _SearchPlan, pools: _Pools, candidates: list[str]-) -> tuple[bool, list[str]]:- """Whether a returned symbol names the query exactly, and the names none does.-- Missing names leave only exact symbols in *pools*.- """- exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False- missing = _missing_named_symbols(- candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts- )- if missing:- pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]- pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]- return exact, missing---def _attach_match_notes(- response: dict,- candidates: list[str],- exact: bool,- missing: list[str],- grep_hint: str | None,-) -> None:- """Exact-match flag, its note, and the grep fallback for an empty window."""- results = response["results"]- # Fuzzy neighbours of a missing name look authoritative, so say so.- if candidates:- response["exact_match"] = exact and not missing- note = _exact_match_note(- candidates,- exact,- missing,- response.get("fuzzy_omitted", 0),- bool(results),- grep_hint,- )- if note:- response["note"] = note- if grep_hint and not results:- response["grep_hint"] = grep_hint---async def _structured_search(- query: str,- limit: int,- page_type: str | None,- kind: str | None,- symbol_kind: str | None,- repo: str | None,- mode: str,- grep_hint: str | None,- names: Container[str] | None = None,- fed: Federation | None = None,-) -> dict:- """Run symbol / path / hybrid search, across repos when federated, and shape the response.-- ``grep_hint`` is attached only when nothing was found.- """- multi = fed is not None and fed.multi- canonical_symbol = _canonical_symbol_query(query)- plan = _SearchPlan(- query=query,- limit=limit,- page_type=page_type,- kind=kind,- symbol_kind=symbol_kind,- mode=mode,- names=names,- canonical_symbol=canonical_symbol,- symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),- fetch=_symbol_fetch(mode, limit),- multi=multi,- )-- async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:- return await _search_one_repo(ctx, plan)-- pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)- candidates = _identifier_candidates(query, mode, names)- _protect_exact(plan, pools, candidates)- exact, missing = _match_named(plan, pools, candidates)-- results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)- repository = await _single_repository(pools.contexts, multi)-- response: dict = {- "results": results,- "mode": mode,- "_meta": _build_meta(repository=repository, targets=_result_paths(results)),- }- if fuzzy_omitted:- response["fuzzy_omitted"] = fuzzy_omitted- # Not ``candidates``: that name holds the query's identifiers the note quotes.- if file_cands := file_candidates(results, limit=limit):- response["candidates"] = file_cands- _attach_match_notes(response, candidates, exact, missing, grep_hint)- if fed is not None:- fed.annotate(response, results)- # Last: slims fields the steps above still read.- _slim_served_rows(- results,- {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},- )- _serve_snippets(results)- return response--- surface_order=40,- artifact_type="search_results",- presentation="search_results",- evidence_basis="inferred",- recipes=(- ToolRecipe(- "find_raw_hits",- 'search_codebase(query="identifier or path", mode="auto")',- ("search_codebase",),- ),- ),-)-async def search_codebase(- query: str = "",- limit: int = 5,- page_type: str | None = None,- kind: str | None = None,- repo: str | None = None,- mode: str = "auto",- symbol_kind: str | None = None,- pattern: str | None = None,-) -> dict:- """Find code by concept, symbol, or path — hybrid codebase search.-- For questions (how/where/why), call get_answer instead: it runs this- retrieval internally and returns a cited answer. Use this for raw- ranked hits: enumerating matches, resolving an identifier to a symbol_id,- or scoping get_context.-- mode="auto" routes by query shape: identifiers search symbols (symbol_id,- path, line bounds for get_symbol), paths resolve files (for get_context),- prose runs wiki-semantic search, and mixed queries run hybrid (symbol hits,- then file-backed pages only). Decision records rank below file pages- unless the query is why-shaped.-- Rows naming a file carry `path`; concept pages add `symbols`- (name:line) the query matches; the top 3 add `matched_lines`- (line, text), sample lines sharing its words. `candidates` lists up to `limit`- distinct files to Read, best first. Identifier or literal queries also- return `lines` (path, line, kind, text; definitions first) read from live- files; when `complete` is true they are every match, so no grep is needed.-- Args:- query: identifier, path, or natural language.- limit: max results (default 5); distinct files outside mode="symbol"- (same-file symbols in `symbols`).- page_type: file_page (per-file docs) or module_page (subsystem- pages); any stored page type filters.- kind: implementation | test | config | doc (concept/symbol modes).- repo: alias, or "all" for workspace-wide.- mode: auto | concept | symbol | path | hybrid.- symbol_kind: filter symbol hits (function|class|method|...).- pattern: alias for query.- """- ignored: list[dict[str, Any]] = []- # Hosts that defer tool schemas let a model guess grep's argument name.- if pattern is not None and not query.strip():- query = pattern- elif pattern is not None:- ignored.append(- {"argument": "pattern", "values": [pattern], "valid": [], "superseded_by": "query"}- )- if not query.strip():- return {- "results": [],- "error": "search_codebase requires `query` (or its alias `pattern`).",- "_meta": _build_meta(),- }- # An unknown kind used to take the same ``return False`` as a kind that is- # simply inapplicable, so a typo and a real empty result looked identical.- kind = resolve_enum_argument(kind, _VALID_KINDS, argument="kind", ignored=ignored)- # An unknown mode used to become ``auto`` with nothing said, so the caller- # got a different search than it asked for. Modes have always been- # case-insensitive; only a genuine miss is dropped, and under its own spelling.- if mode is not None and mode.lower() in _VALID_MODES:- mode = mode.lower()- mode = resolve_enum_argument(mode, _VALID_MODES, argument="mode", ignored=ignored)- # One fan-out state for the whole call: a repo that fails one stage is- # skipped by the next and reported once.- fed = Federation() if repo == "all" else None- # Loaded once so routing, candidates, ordering and the hint validate alike.- names: set[str] = set()- for _, repo_names in await _per_repo(repo, fed, lambda ctx: indexed_names([ctx], query)):- names |= repo_names- grep_hint = _grep_hint_for(query, names)- resolved_mode = _resolve_mode(query, mode, names)- # Single repo only: a federated list has no one tree to name files in.- issue: IssueFiles | None = None- if repo != "all":- try:- issue = await issue_files(await _resolve_repo_context(repo), query, names)- except Exception:- _log.warning("search_codebase: issue file lookup failed", exc_info=True)-- if resolved_mode in ("symbol", "path", "hybrid"):- structured = await _structured_search(- query, limit, page_type, kind, symbol_kind, repo, resolved_mode, grep_hint, names, fed- )- _lead_candidates(structured, issue, limit)- await attach_line_hits(structured, query, resolved_mode, names, repo)- attach_ignored_arguments(structured, ignored)- return structured-- if fed is not None:- # kind is filtered per-repo inside _search_single_repo, before each- # repo's limit cut, so the fused list is already kind-pure and full.- federated = await _federated_search(query, limit, page_type, kind, fed)- if grep_hint and not federated.get("results"):- federated["grep_hint"] = grep_hint- attach_ignored_arguments(federated, ignored)- return federated-- ctx = await _resolve_repo_context(repo)-- async with get_session(ctx.session_factory) as session:- # Validate repo exists in DB- repository = await _get_repo(session)-- await _wait_for_vector_store(ctx)-- # Fuse FTS + vector via RRF (see _fused_retrieve) so a page either mode- # ranks highly surfaces here, matching get_answer's retrieval. Always- # over-fetch (see _fetch_limit_for): post-filters trim hits, and decision- # down-weighting needs file pages inside the window to promote.- fetch_limit = _fetch_limit_for(limit, kind)- output = await _fused_retrieve(ctx, query, fetch_limit, page_type)-- _downweight_decisions(output, query)- output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)-- # Batch-lookup page target paths for the kind filter + git freshness boost- if output:- async with get_session(ctx.session_factory) as session:- page_info, tombstoned, git_map = await _load_page_info(session, output, with_git=True)- path_index = await file_path_index(session, repository.id)- # Tombstoned pages document deleted/renamed files — never results.- output = [item for item in output if item["page_id"] not in tombstoned]-- # Attach target_path to each item so the kind filter (path-prefix- # heuristic) and downstream get_context callers can act on it, and- # ``file`` wherever the page id is symbol-qualified.- _attach_paths(output, page_info)-- output = filter_dicts_by_key(output, "target_path", _get_exclude_spec(ctx.path))-- for item in output:- # Freshness boost: recently-active files rank higher- target_path = page_info.get(item["page_id"])- gm = git_map.get(target_path) if target_path else None- _apply_freshness_boost(item, gm)-- # Re-sort by adjusted relevance with retrieval noise (decisions on- # non-why queries, test pages on non-test queries) hard-demoted, then- # collapse near-duplicate decisions to one.- output = rerank_pages_first(output, query, score_key="relevance_score", floor=0.5)- # After the coverage rerank, whose window-relative weights cannot tell- # a word rare across the repo from one rare in these few hits.- boost_named_paths(- output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"- )- _downweight_test_pages(output, query)- _sort_demoting_noise(output, query)- output = _dedup_decisions(output)-- output = _filter_by_kind(output, kind)- # A file page and a symbol page of one file are one place to look.- output = _collapse_by_file(output)- # Files the page retrievers structurally cannot see (a private helper, a- # local name, anything a file page's public-symbol table omits) get the- # weakest tail slots. No-op when the symbol leg names nothing new.- output = _reserve_symbol_tail(output, limit, kind)- _drop_internal_ranking_fields(output)- # The full ranked pool, kept before the caller's cut so ``candidates``- # below can reach past it. See its comment for why that matters.- ranked = list(output)- output = output[:limit]- await attach_hit_symbols(ctx, query, output)-- # Derive confidence_score from relative position in the result set.- _assign_confidence(output, "relevance_score", "confidence_score")-- response: dict = {- "results": output,- "_meta": _build_meta(repository=repository, targets=_result_paths(output)),- }- if candidates := file_candidates(ranked, limit=limit):- response["candidates"] = candidates- _lead_candidates(response, issue, limit)- if grep_hint and not output:- response["grep_hint"] = grep_hint- await attach_line_hits(response, query, resolved_mode, names, repo)- attach_ignored_arguments(response, ignored)- # Last, so nothing above has to know the field is on its way out.- _slim_served_rows(output, {None: _page_language(ctx)})- _serve_snippets(output)- return responsediff --git a/packages/server/src/repowise/server/mcp_server/tool_search/__init__.py b/packages/server/src/repowise/server/mcp_server/tool_search/__init__.pynew file mode 100644index 000000000..bb3109a03--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/__init__.py@@ -0,0 +1,186 @@+"""MCP Tool 5: search_codebase, hybrid codebase search.++``tool.py`` holds the registered tool; the sibling modules hold its stages.+Importing the package registers the tool. Every name the single module had+is re-exported here, and a stage calls a name tests patch through the package+(``_pkg.<name>``), never by bare name, or the patch stops reaching it.+"""++# ruff: noqa: F401++from __future__ import annotations++from sqlalchemy import select++from repowise.core.generation.structural_labels import is_structural_title+from repowise.core.persistence.database import get_session+from repowise.core.persistence.models import (+ GitMetadata,+ Page,+)+from repowise.core.persistence.search import strip_leading_headings+from repowise.core.providers.embedding import store_has_semantic_vectors+from repowise.core.registry import ToolRecipe+from repowise.core.registry import mcp_tool_registry as mcp+from repowise.core.test_paths import is_test_path, is_test_related_path+from repowise.server.mcp_server._answer_pipeline import (+ _FILENAME_LEG_RRF_K,+ _RRF_K,+ _RRF_SCORE_SCALE,+ _SYMBOL_LEG_RRF_K,+ _safe_filename_search,+ _safe_symbol_search,+ leg_ranks,+)+from repowise.server.mcp_server._budget import (+ OmissionCollector,+ register_post_enforce,+ register_post_shed,+)+from repowise.server.mcp_server._federation import Federation, merge_ranked+from repowise.server.mcp_server._helpers import (+ _VECTOR_TIMEOUT_ENV,+ _get_exclude_spec,+ _get_repo,+ _resolve_repo_context,+ attach_ignored_arguments,+ filter_dicts_by_key,+ resolve_enum_argument,+ vector_search_timeout_s,+)+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols+from repowise.server.mcp_server._line_hits import attach_line_hits+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT, semantic_search_state+from repowise.server.mcp_server._meta import build_meta as _build_meta+from repowise.server.mcp_server._page_paths import (+ FILE_ROW_TYPES,+ PAGELESS_FILE,+ add_row_paths,+ file_candidates,+ file_path_of,+ hit_file_path,+ pageless_path,+)+from repowise.server.mcp_server._query_shape import (+ _DECISION_DOWNWEIGHT,+ _MIN_RELEVANCE_SCORE,+ _VALID_MODES,+ NOT_THE_NAMED_SYMBOL,+ _canonical_symbol_query,+ _embedded_identifiers,+ _fetch_limit_for,+ _has_exact_symbol,+ _identifier_candidates,+ _is_why_shaped,+ _looks_like_code_name,+ _looks_like_exact_token,+ _mark_not_the_named_symbol,+ _names_a_path,+ _qual_norm,+ _resolve_mode,+ _symbol_matches_name,+ path_tokens,+)+from repowise.server.mcp_server._query_terms import content_terms+from repowise.server.mcp_server._references import path_identity, symbol_identity+from repowise.server.mcp_server._retrieval_rank import (+ boost_named_paths,+ rerank_pages_first,+)+from repowise.server.mcp_server.tool_search.ordering import (+ _FILE_WINDOW_OVERFETCH,+ _MERGED_SYMBOL_CAP,+ _collapse_by_file,+ _hybrid_window,+ _interleave_hybrid,+ _prose_dominates,+ _protect_exact_paths,+ _protect_exact_symbols,+ _protect_named_symbols,+)+from repowise.server.mcp_server.tool_search.ranking import (+ _CONFIG_PATH_TOKENS,+ _SYMBOL_TAIL_DIVISOR,+ _TEST_DOWNWEIGHT,+ _TEST_QUERY_RE,+ _VALID_KINDS,+ _classify_hit_kind,+ _dedup_decisions,+ _downweight_decisions,+ _downweight_test_pages,+ _filter_by_kind,+ _is_test_page,+ _is_test_query,+ _non_decision_fallback,+ _norm_decision_title,+ _rescue_all_decision_window,+ _reserve_symbol_tail,+ _sort_demoting_noise,+ _symbol_tail_reserve,+)+from repowise.server.mcp_server.tool_search.retrieval import (+ _federated_search,+ _fused_entry,+ _fused_retrieve,+ _neg_score,+ _per_repo,+ _safe_fts,+ _safe_vector,+ _search_single_repo,+ _symbol_leg_target,+ _wait_for_vector_store,+)+from repowise.server.mcp_server.tool_search.serve import (+ _FRESHNESS_TIEBREAK,+ _apply_freshness_boost,+ _assign_confidence,+ _attach_paths,+ _drop_derivable_page_ids,+ _drop_internal_ranking_fields,+ _grep_hint_for,+ _load_page_info,+ _page_language,+ _refresh_served_targets,+ _refresh_served_targets_final,+ _result_paths,+ _serve_snippets,+ _slim_served_rows,+)+from repowise.server.mcp_server.tool_search.structured import (+ _attach_match_notes,+ _exact_match_note,+ _hybrid_mode_results,+ _lead_candidates,+ _match_named,+ _merge_pools,+ _missing_named_symbols,+ _Pools,+ _protect_exact,+ _search_one_repo,+ _SearchPlan,+ _select_results,+ _single_repository,+ _structured_search,+ _symbol_fetch,+ _symbol_mode_results,+ _symbol_query_for,+ _tag_repo,+)+from repowise.server.mcp_server.tool_search.tool import (+ _concept_search,+ _indexed_names,+ _issue_files,+ _query_from_pattern,+ _rank_concept_pages,+ _resolve_kind_and_mode,+ search_codebase,+)+from repowise.server.mcp_server.tool_search_symbols import (+ _MAX_CANDIDATES,+ IssueFiles,+ file_path_index,+ indexed_names,+ issue_files,+ search_paths_single,+ search_symbols_single,+)diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/ordering.py b/packages/server/src/repowise/server/mcp_server/tool_search/ordering.pynew file mode 100644index 000000000..a73aae833--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/ordering.py@@ -0,0 +1,158 @@+"""Exact-match protection and hybrid window ordering for search results."""++from __future__ import annotations++import re+from collections.abc import Container, Sequence++from repowise.server.mcp_server._page_paths import hit_file_path+from repowise.server.mcp_server._query_shape import (+ _canonical_symbol_query,+ _embedded_identifiers,+ _qual_norm,+ _symbol_matches_name,+ path_tokens,+)+++def _protect_exact_symbols(query: str, symbols: list[dict]) -> list[dict]:+ """Stable-partition an exact identifier or ``path::Symbol`` hit to the head."""+ if not symbols:+ return symbols+ canonical = _canonical_symbol_query(query)+ qnorm = query.strip().lower().replace("\\", "/")++ def exact(item: dict) -> bool:+ symbol_id = (item.get("symbol_id") or "").lower().replace("\\", "/")+ if canonical:+ return symbol_id == qnorm+ return qnorm in {+ (item.get("name") or "").lower(),+ _qual_norm(item.get("qualified_name")),+ }++ protected = [item for item in symbols if exact(item)]+ return protected + [item for item in symbols if not exact(item)]+++def _protect_named_symbols(candidates: list[str], symbols: list[dict]) -> list[dict]:+ """Stable-partition symbols exactly named by identifiers embedded in prose."""+ wanted = {candidate.strip().lower() for candidate in candidates if candidate.strip()}+ wanted |= {_qual_norm(candidate) for candidate in candidates if candidate.strip()}++ protected = [item for item in symbols if _symbol_matches_name(item, wanted)]+ return protected + [item for item in symbols if not _symbol_matches_name(item, wanted)]+++def _protect_exact_paths(+ query: str, files: list[dict], paths: Sequence[str] = ()+) -> list[dict]:+ """Stable-partition an exact path hit ahead of basename/fuzzy neighbours."""+ tokens = path_tokens(query, paths) or [query]+ wanted = {t.strip().lower().replace("\\", "/") for t in tokens}+ protected = [item for item in files if (item.get("file") or "").lower() in wanted]+ return protected + [item for item in files if (item.get("file") or "").lower() not in wanted]+++def _prose_dominates(query: str, identifiers: list[str]) -> bool:+ """True when natural-language tokens outnumber the identifier tokens a query+ carries: the query reads as prose that merely *mentions* a symbol, not a+ symbol lookup dressed in a few words. Drives hybrid ordering below."""+ ident_count = len(identifiers)+ if ident_count == 0:+ return False+ total = len(re.findall(r"[A-Za-z0-9_]+", query))+ return (total - ident_count) > ident_count+++def _interleave_hybrid(+ query: str,+ symbols: list[dict],+ concepts: list[dict],+ limit: int,+ exact: bool,+ names: Container[str] | None = None,+) -> list[dict]:+ """Order one hybrid result window from the two incomparable score scales.++ Symbol SQL scores (~43, +100 on an exact-name hit) and concept relevance+ (0 to 1) can't be merge-sorted, so we interleave by block. Default: symbols+ lead, reserving up to half the window for concept pages so a flood of symbol+ matches can't truncate every page out.++ The exception is the score-scale trap. When NO returned symbol matches the+ query's identifier exactly AND the query is mostly prose, leading with fuzzy+ symbol hits buries the page the caller actually wants: the generic ``.get``+ methods that outscore the ``answer.py`` page for "how does retrieval feed+ synthesis in get_answer". There, concept pages lead and the fuzzy symbols+ fall to the tail (nothing is dropped, only reordered within the window).+ """+ if not exact and concepts and _prose_dominates(query, _embedded_identifiers(query, names)):+ reserved = min(len(symbols), limit // 2)+ return (concepts[: max(1, limit - reserved)] + symbols)[:limit]+ reserved = min(len(concepts), limit // 2)+ return (symbols[: max(1, limit - reserved)] + concepts)[:limit]+++# Code-location windows over-fetch this many times ``limit`` so collapsing+# same-file hits still leaves ``limit`` distinct files to serve.+_FILE_WINDOW_OVERFETCH = 2+# Other symbol names a collapsed row carries in ``symbols``.+_MERGED_SYMBOL_CAP = 5+++def _collapse_by_file(hits: list[dict]) -> list[dict]:+ """One row per (repo, file), best first; rows naming no file pass through.++ The first row of a file wins. A later symbol hit in the same file folds+ into the winner's ``symbols`` as ``name:line`` (capped) instead of taking a+ slot of its own, so ``limit`` buys distinct files; past the cap a trailing+ ``+N more`` entry counts the rest. Precedent:+ ``symbol_backed_pages`` collapses the same way for the concept tail.+ """+ first_of: dict[tuple, dict] = {}+ extra: dict[int, int] = {}+ out: list[dict] = []+ for hit in hits:+ path = hit_file_path(hit)+ if path is None:+ out.append(hit)+ continue+ key = (hit.get("repo"), path)+ first = first_of.get(key)+ if first is None:+ first_of[key] = hit+ out.append(hit)+ elif hit.get("type") == "symbol" and first.get("type") == "symbol":+ merged = first.setdefault("symbols", [])+ if len(merged) < _MERGED_SYMBOL_CAP:+ merged.append(f"{hit.get('name')}:{hit.get('start_line')}")+ else:+ extra[id(first)] = extra.get(id(first), 0) + 1+ for hit in out:+ if id(hit) in extra:+ hit["symbols"].append(f"+{extra[id(hit)]} more")+ return out+++def _hybrid_window(+ query: str,+ symbols: list[dict],+ concepts: list[dict],+ limit: int,+ exact: bool,+ names: Container[str] | None = None,+) -> list[dict]:+ """The hybrid window, with no page for a file a shown symbol row covers.++ Pages are dropped only for symbols the window actually shows, then the+ window is rebuilt so the freed slots backfill. Concepts only shrink and the+ symbol share only grows, so this settles in a few passes.+ """+ while True:+ window = _interleave_hybrid(query, symbols, concepts, limit, exact, names)+ shown = {item.get("file") for item in window if item.get("type") == "symbol"}+ kept = [c for c in concepts if c.get("target_path") not in shown]+ if len(kept) == len(concepts):+ return window+ concepts = keptdiff --git a/packages/server/src/repowise/server/mcp_server/tool_search/ranking.py b/packages/server/src/repowise/server/mcp_server/tool_search/ranking.pynew file mode 100644index 000000000..2fd52cf49--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/ranking.py@@ -0,0 +1,287 @@+"""Noise demotion, decision rescue, symbol tail and kind filtering."""++from __future__ import annotations++import asyncio+import contextlib+import logging+import re++from repowise.core.test_paths import is_test_path, is_test_related_path++# Patchable names are read off the package at call time so patches reach them.+from repowise.server.mcp_server import tool_search as _pkg+from repowise.server.mcp_server._helpers import vector_search_timeout_s+from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES+from repowise.server.mcp_server._query_shape import (+ _DECISION_DOWNWEIGHT,+ _MIN_RELEVANCE_SCORE,+ _is_why_shaped,+)++_log = logging.getLogger("repowise.mcp.search")+++def _downweight_decisions(output: list[dict], query: str) -> None:+ """Scale decision_record relevance in place unless the query is why-shaped."""+ if _is_why_shaped(query):+ return+ for item in output:+ if item.get("page_type") == "decision_record" and item.get("relevance_score"):+ item["relevance_score"] = round(item["relevance_score"] * _DECISION_DOWNWEIGHT, 4)+++# Test pages win retrieval on vocabulary shared with the code they exercise, but+# are rarely a better first Read: demote them unless the query is about tests.+_TEST_DOWNWEIGHT = 0.6++_TEST_QUERY_RE = re.compile(+ r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",+ re.IGNORECASE,+)+++def _is_test_query(query: str) -> bool:+ """True when the query is explicitly about tests, so test pages rank naturally."""+ return bool(_TEST_QUERY_RE.search(query))+++def _is_test_page(item: dict) -> bool:+ """True when a hit is a whole-file row for a test file.++ Tests only, deliberately not test support: a ``conftest.py`` or a fixture+ module is often exactly what the person searching wanted, so it keeps its+ rank. The ``kind`` filter counts both (see :func:`_classify_hit_kind`) —+ asking for tests and asking to rank tests lower are different questions.+ """+ return item.get("page_type") in FILE_ROW_TYPES and is_test_path(item.get("target_path") or "")+++def _downweight_test_pages(output: list[dict], query: str) -> None:+ """Scale test file_page relevance in place unless the query is about tests."""+ if _is_test_query(query):+ return+ for item in output:+ if item.get("relevance_score") and _is_test_page(item):+ item["relevance_score"] = round(item["relevance_score"] * _TEST_DOWNWEIGHT, 4)+++def _sort_demoting_noise(output: list[dict], query: str) -> None:+ """Sort by relevance, ranking retrieval noise below every real page.++ Two classes crowd the top ranks on a plain implementation query and are+ demoted absolutely for their query class (the relevance score still orders+ each class among itself):++ - decision records — short dense titles win cosine similarity by a margin+ wider than the multiplicative down-weight. Rationale questions are+ get_why's territory, so a why-shaped query ranks them naturally instead.+ - test file pages — a test is rarely a better first Read than the code it+ exercises. A query explicitly about tests ranks them naturally instead.+ """+ why = _is_why_shaped(query)+ test_focused = _is_test_query(query)++ def key(item: dict) -> tuple:+ pt = item.get("page_type")+ is_decision = (not why) and pt == "decision_record"+ is_test = (not test_focused) and _is_test_page(item)+ return (1 if (is_decision or is_test) else 0, -(item.get("relevance_score") or 0.0))++ output.sort(key=key)+++def _norm_decision_title(title: str) -> str:+ """Normalize a decision title to collapse near-duplicate phrasings."""+ return re.sub(r"[^a-z0-9]+", " ", (title or "").lower()).strip()+++def _dedup_decisions(output: list[dict]) -> list[dict]:+ """Collapse near-duplicate decision records by normalized title (order-preserving).++ Callers dedup AFTER the score sort, so the surviving variant is the+ highest-scored one; non-decision pages always pass through untouched.+ """+ seen: set[str] = set()+ out: list[dict] = []+ for item in output:+ if item.get("page_type") == "decision_record":+ key = _norm_decision_title(item.get("title", ""))+ if key and key in seen:+ continue+ seen.add(key)+ out.append(item)+ return out+++async def _non_decision_fallback(ctx, query: str, fetch_limit: int) -> list[dict]:+ """Wider re-fetch keeping only non-decision pages.++ Guard for the window-saturation failure: when the entire over-fetched+ window is decision records, demotion has nothing to promote and a+ non-why query returns zero implementation pages. Fetch 4x wider, drop+ decisions, and let the caller merge the survivors.+ """+ results = []+ src = "vector"+ # Skipped on a keyless index, which then falls through to the FTS branch.+ if _pkg.store_has_semantic_vectors(ctx.vector_store):+ try:+ results = await asyncio.wait_for(+ ctx.vector_store.search(query, limit=fetch_limit * 4),+ timeout=vector_search_timeout_s(),+ )+ except TimeoutError:+ _log.warning("Vector re-fetch timed out; falling back to full-text")+ except Exception:+ _log.debug("Vector re-fetch failed; falling back to full-text", exc_info=True)+ if not results:+ src = "fts"+ with contextlib.suppress(Exception):+ results = await ctx.fts.search(query, limit=fetch_limit * 4)++ out = []+ for r in results:+ if r.page_type == "decision_record":+ continue+ if r.score < _MIN_RELEVANCE_SCORE:+ continue+ out.append(+ {+ "page_id": r.page_id,+ "title": r.title,+ "page_type": r.page_type,+ "snippet": r.snippet,+ "relevance_score": r.score,+ "sources": [src],+ }+ )+ return out+++async def _rescue_all_decision_window(+ ctx, output: list[dict], query: str, fetch_limit: int+) -> list[dict]:+ """Merge non-decision pages into an all-decision result window.++ No-op unless the query is non-why AND every current hit is a decision+ record. Deduplicates by page_id.+ """+ if not output or _is_why_shaped(query):+ return output+ if not all(item.get("page_type") == "decision_record" for item in output):+ return output+ fallback = await _non_decision_fallback(ctx, query, fetch_limit)+ seen = {item["page_id"] for item in output}+ output.extend(item for item in fallback if item["page_id"] not in seen)+ return output+++# Slots at the tail of a concept window reserved for files reached through the+# symbol index rather than through their page. A generated file page renders+# only public symbols, so a question about a private helper or a local name has+# nothing to match in either the full-text index or the embedding. The symbol+# index is where those names live. One slot in three, taken from the weakest+# end of the window, is a cheap price for reaching a class of file the page+# retrievers structurally cannot see. No-op when the symbol leg finds nothing.+_SYMBOL_TAIL_DIVISOR = 3+++def _symbol_tail_reserve(limit: int) -> int:+ """How many tail slots the symbol leg may take from a window of *limit*."""+ return limit // _SYMBOL_TAIL_DIVISOR+++def _reserve_symbol_tail(output: list[dict], limit: int, kind: str | None) -> list[dict]:+ """Give the weakest window slots to symbol-leg files ranked past ``limit``.++ Concept hits keep the head. Files the symbol leg reached that fusion left+ outside the window take up to :func:`_symbol_tail_reserve` slots, and the+ concept hits they displaced fall in behind them (nothing is dropped).+ """+ reserve = _symbol_tail_reserve(limit)+ if reserve <= 0 or kind in ("config", "doc"):+ # config/doc windows are asking for pages with no symbols in them.+ return output+ extra = [item for item in output[limit:] if "symbol" in item.get("sources", ())][:reserve]+ if not extra:+ return output+ head = output[: limit - len(extra)]+ taken = {id(item) for item in head + extra}+ return head + extra + [item for item in output if id(item) not in taken]+++# Path-prefix heuristics for the ``kind`` filter. We classify a hit's+# target_path against these prefixes; if none match, the hit falls into+# ``other`` and is dropped only when the caller asked for a specific kind.+# Tests are not in this list: they come from ``repowise.core.test_paths``, the+# same rules that stamped the file's ``is_test`` flag at ingestion.+_CONFIG_PATH_TOKENS = (+ "pyproject.toml",+ "package.json",+ "tsconfig",+ "setup.py",+ "setup.cfg",+ "/.github/",+ "dockerfile",+ ".yml",+ ".yaml",+ ".toml",+ ".ini",+ ".cfg",+ "lockfile",+ "package-lock",+ "uv.lock",+ "poetry.lock",+)+++def _classify_hit_kind(target_path: str, page_type: str) -> str:+ """Bucket a hit into implementation / test / config / doc.++ Pages without a file behind them (decision records, repo overviews,+ onboarding pages — all have empty ``target_path``) are docs: letting+ them fall through to the path heuristics classified every decision+ record as "implementation", so ``kind="implementation"`` returned+ decision pages instead of filtering them out.++ ``test`` is the union of tests and test support here: someone asking for+ ``kind="implementation"`` does not want a ``conftest.py`` back, and someone+ asking for ``kind="test"`` does.++ Config wins over test, which the token list this replaced never had to+ decide: ``.github/workflows/tests.yml`` is a workflow whatever it is named,+ and dropping it from ``kind="config"`` would be the more surprising answer.+ """+ tp = (target_path or "").lower()+ if page_type in ("module_page", "symbol_spotlight") or tp.endswith(".md"):+ return "doc"+ if not tp or page_type not in FILE_ROW_TYPES:+ return "doc"+ if any(tok in tp for tok in _CONFIG_PATH_TOKENS):+ return "config"+ # Original case, not ``tp``: the camel (FooTest.java) and .NET project-dir+ # (Foo.Tests/) rules are deliberately case-sensitive.+ if is_test_related_path(target_path):+ return "test"+ return "implementation"+++# Every value :func:`_classify_hit_kind` can return; anything else can only ever+# match nothing, which is why an unknown kind is dropped rather than filtered on.+_VALID_KINDS = frozenset({"implementation", "test", "config", "doc"})+++def _filter_by_kind(output: list[dict], kind: str | None) -> list[dict]:+ """Keep only hits classified as ``kind`` (no-op when ``kind`` is falsy).++ Runs on the over-fetched list BEFORE the limit cut so the caller still+ gets up to ``limit`` results of the requested kind.+ """+ if not kind:+ return output+ return [+ item+ for item in output+ if _classify_hit_kind(item.get("target_path", ""), item.get("page_type", "")) == kind+ ]diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py b/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.pynew file mode 100644index 000000000..62bb71207--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py@@ -0,0 +1,307 @@+"""Fused page retrieval for one repo and across a workspace."""++from __future__ import annotations++import asyncio+import contextlib+import logging+from typing import Any++from repowise.core.persistence.database import get_session++# Patchable names are read off the package at call time so patches reach them.+from repowise.server.mcp_server import tool_search as _pkg+from repowise.server.mcp_server._answer_pipeline import (+ _FILENAME_LEG_RRF_K,+ _RRF_K,+ _RRF_SCORE_SCALE,+ _SYMBOL_LEG_RRF_K,+ leg_ranks,+)+from repowise.server.mcp_server._federation import Federation, merge_ranked+from repowise.server.mcp_server._helpers import (+ _VECTOR_TIMEOUT_ENV,+ _get_repo,+ _resolve_repo_context,+ filter_dicts_by_key,+ vector_search_timeout_s,+)+from repowise.server.mcp_server._meta import build_meta as _build_meta+from repowise.server.mcp_server._page_paths import (+ PAGELESS_FILE,+ file_candidates,+ file_path_of,+ pageless_path,+)+from repowise.server.mcp_server._query_shape import (+ _MIN_RELEVANCE_SCORE,+ _fetch_limit_for,+)+from repowise.server.mcp_server._retrieval_rank import boost_named_paths+from repowise.server.mcp_server.tool_search_symbols import file_path_index++from .ordering import _collapse_by_file+from .ranking import (+ _dedup_decisions,+ _downweight_decisions,+ _downweight_test_pages,+ _filter_by_kind,+ _rescue_all_decision_window,+ _sort_demoting_noise,+)+from .serve import (+ _assign_confidence,+ _attach_paths,+ _load_page_info,+ _page_language,+ _serve_snippets,+ _slim_served_rows,+)++_log = logging.getLogger("repowise.mcp.search")+++async def _wait_for_vector_store(ctx) -> None:+ """Block (bounded) until the vector store signals readiness, if it tracks it."""+ if ctx.vector_store_ready is not None:+ with contextlib.suppress(TimeoutError):+ await asyncio.wait_for(ctx.vector_store_ready.wait(), timeout=30.0)+++async def _safe_fts(ctx, query: str, limit: int) -> list:+ """FTS search, bounded and failure-swallowing (returns [] on error/timeout)."""+ if ctx.fts is None:+ return []+ with contextlib.suppress(Exception):+ return await asyncio.wait_for(ctx.fts.search(query, limit=limit), timeout=5.0)+ return []+++async def _safe_vector(ctx, query: str, limit: int) -> list:+ """Vector search, bounded and failure-swallowing (returns [] on error/timeout).++ Readiness is awaited by the caller (``_wait_for_vector_store``) before the+ fused retrieve runs, so this path does not re-wait.++ Returns nothing on a keyless index. ``store_has_semantic_vectors`` explains+ why the mock's vectors cannot be ranked on; the reason this is enforced here+ rather than in ``_fused_retrieve`` is that this function is the only place+ the vector leg is entered, so a later caller inherits the guard.+ """+ if ctx.vector_store is None or not _pkg.store_has_semantic_vectors(ctx.vector_store):+ return []+ try:+ return await asyncio.wait_for(+ ctx.vector_store.search(query, limit=limit), timeout=vector_search_timeout_s()+ )+ except TimeoutError:+ # Logged, not silent: a timeout drops the whole semantic leg and the+ # full-text-only hits that remain look like a normal result set.+ _log.warning(+ "Vector search exceeded its %gs budget; returning no semantic hits. "+ "Raise it with %s=<seconds>.",+ vector_search_timeout_s(),+ _VECTOR_TIMEOUT_ENV,+ )+ except Exception:+ _log.debug("Vector search failed; returning no semantic hits", exc_info=True)+ return []+++def _symbol_leg_target(fused: dict[str, dict], page_id: str) -> str:+ """The fused page a symbol-leg file page credits: the best page of its file.++ The leg names files, while the page legs may have reached the same file+ through a symbol page. Crediting only the file page id would leave that+ hit unlifted and add a weak twin that collapsing then drops.+ """+ path = page_id.partition(":")[2]+ best = None+ for pid, entry in fused.items():+ if file_path_of(entry["page_type"], pid.partition(":")[2]) != path:+ continue+ if best is None or entry["_rrf"] > fused[best]["_rrf"]:+ best = pid+ return best or page_id+++def _fused_entry(r) -> dict:+ """Seed a fused-result dict from a retriever hit (RRF score added by caller)."""+ entry = {+ "page_id": r.page_id,+ "title": r.title,+ "page_type": r.page_type,+ "snippet": r.snippet,+ "_rrf": 0.0,+ "_sources": set(),+ }+ if r.page_type == PAGELESS_FILE:+ # No page row to load it from later.+ entry["target_path"] = pageless_path(r.page_id)+ return entry+++async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | None) -> list[dict]:+ """Retrieve by fusing FTS, vector and symbol search via Reciprocal Rank Fusion.++ The retrievers run in parallel; a hit's score is the sum of+ ``1/(rank + k)`` over the retrievers that surfaced it, scaled into the+ BM25 range the downstream relevance gates were tuned against. Each hit+ carries a ``sources`` list (``"fts"``, ``"vector"``, ``"symbol"``,+ ``"filename"``): a page found by several retrievers is a stronger match+ than one found by one mode. The symbol and filename legs also name files+ that have no page, as ``page_type: "file"`` rows.++ This mirrors get_answer's ``hybrid_retrieve`` so the two entry points rank+ the same corpus identically. The prior path ran vector search and fell+ back to FTS only when vector returned nothing, so a page FTS ranked highly+ but vector missed never surfaced here.++ A hit is admitted only when its raw retriever score clears+ ``_MIN_RELEVANCE_SCORE`` — the fused score is rank-based and would+ otherwise sit well above the floor for every window hit, letting an+ off-topic query return its nearest-but-unrelated neighbours. Rank position+ (for RRF) is the hit's place in the retriever's own list, unchanged by the+ floor.+ """+ fts_results, vec_results, sym_results, name_results = await asyncio.gather(+ _pkg._safe_fts(ctx, query, fetch_limit),+ _pkg._safe_vector(ctx, query, fetch_limit),+ _pkg._safe_symbol_search(ctx, query, pageless=True),+ _pkg._safe_filename_search(ctx, query),+ )++ fused: dict[str, dict] = {}+ for rank, r in enumerate(vec_results):+ if r.score < _MIN_RELEVANCE_SCORE:+ continue+ entry = fused.setdefault(r.page_id, _fused_entry(r))+ entry["_rrf"] += 1.0 / (rank + _RRF_K)+ entry["_sources"].add("vector")+ for rank, r in enumerate(fts_results):+ if r.score < _MIN_RELEVANCE_SCORE:+ continue+ entry = fused.setdefault(r.page_id, _fused_entry(r))+ entry["_rrf"] += 1.0 / (rank + _RRF_K)+ entry["_sources"].add("fts")+ # Files whose symbol names the query's words spell, paged or not. Weighted+ # below the page legs, as in get_answer; it has no raw score to floor.+ for rank, r in leg_ranks(sym_results):+ entry = fused.setdefault(_symbol_leg_target(fused, r.page_id), _fused_entry(r))+ entry["_rrf"] += 1.0 / (rank + _SYMBOL_LEG_RRF_K)+ entry["_sources"].add("symbol")+ # Files whose file name the query's words spell, weighted as a name match.+ # It adds files the other legs missed and never reorders a page they found.+ # A name of three or more words the query spells in full weighs like a page hit.+ for rank, (r, full) in leg_ranks(name_results):+ target = _symbol_leg_target(fused, r.page_id)+ if target in fused and r.page_type != PAGELESS_FILE:+ continue+ entry = fused.setdefault(target, _fused_entry(r))+ entry["_rrf"] += 1.0 / (rank + (_RRF_K if full else _FILENAME_LEG_RRF_K))+ entry["_sources"].add("filename")++ output: list[dict] = []+ for entry in fused.values():+ if page_type and entry["page_type"] != page_type:+ continue+ entry["relevance_score"] = round(entry.pop("_rrf") * _RRF_SCORE_SCALE, 4)+ entry["sources"] = sorted(entry.pop("_sources"))+ output.append(entry)+ output.sort(key=lambda item: item["relevance_score"], reverse=True)+ return output+++async def _search_single_repo(+ ctx, query: str, limit: int, page_type: str | None, kind: str | None = None+) -> list[dict]:+ """Run search against a single repo context.++ Fuses FTS and vector retrieval via RRF (see ``_fused_retrieve``), so each+ hit carries a ``sources`` list naming the retrievers that surfaced it+ rather than a single backend label.++ The ``kind`` filter runs here, on the over-fetched list and BEFORE the+ limit cut — filtering after per-repo truncation returned fewer than+ ``limit`` results (frequently zero) in the federated path.+ """+ await _pkg._wait_for_vector_store(ctx)++ fetch_limit = _fetch_limit_for(limit, kind)+ output = await _fused_retrieve(ctx, query, fetch_limit, page_type)++ _downweight_decisions(output, query)+ output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)++ # Attach target_path and drop excluded hits per repo (so federated search+ # honours each repo's own exclude_patterns) before ranking. Runs on the+ # over-fetched list so the kind filter below has real headroom. Load+ # precedes the sort/demotion so they see the page metadata they key on+ # (test demotion needs target_path to classify a test file page).+ if output:+ async with get_session(ctx.session_factory) as session:+ page_info, tombstoned, _ = await _load_page_info(session, output)+ path_index = await file_path_index(session, (await _get_repo(session)).id)+ output = [item for item in output if item["page_id"] not in tombstoned]+ _attach_paths(output, page_info)+ output = filter_dicts_by_key(output, "target_path", _pkg._get_exclude_spec(ctx.path))+ boost_named_paths(+ output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"+ )++ _downweight_test_pages(output, query)+ _sort_demoting_noise(output, query)+ output = _dedup_decisions(output)++ output = _filter_by_kind(output, kind)+ return _collapse_by_file(output)[:limit]+++def _neg_score(key: str):+ """Best-first sort key on a row's *key* score, for :func:`merge_ranked`."""+ return lambda item: -(item.get(key) or 0.0)+++async def _federated_search(+ query: str, limit: int, page_type: str | None, kind: str | None, fed: Federation+) -> dict:+ """Search across all repos using Reciprocal Rank Fusion."""++ async def _one_repo(ctx) -> list[dict]:+ repo_results = await _search_single_repo(ctx, query, limit, page_type, kind)+ for rank, item in enumerate(repo_results):+ item["repo"] = ctx.alias+ item["rrf_score"] = 1.0 / (rank + 60) # RRF constant k=60+ return repo_results++ answered = await fed.run(_one_repo)+ contexts = [ctx for ctx, _ in answered]+ all_results = merge_ranked(+ (rows for _, rows in answered), _neg_score("rrf_score"), per_repo_cap=limit+ )+ output = all_results[:limit]++ # Derive confidence from RRF position+ _assign_confidence(output, "rrf_score", "confidence_score")++ # No single indexed commit to compare a live HEAD against: per-repo+ # freshness rides on ``repos_stale`` and the rows' ``stale`` flag instead.+ response: dict = {"results": output, "_meta": _build_meta()}+ fed.annotate(response, output)+ # Drawn from every repo's ranked list, not the merged cut: a workspace+ # search that spends its window on module pages should still be able to+ # name files.+ if candidates := file_candidates(all_results, limit=limit):+ response["candidates"] = candidates+ # Last, so nothing above has to know the field is on its way out.+ _slim_served_rows(output, {ctx.alias: _page_language(ctx) for ctx in contexts})+ _serve_snippets(output)+ return response+++async def _per_repo(repo: str | None, fed: Federation | None, fn) -> list[tuple[Any, Any]]:+ """``(context, fn(context))`` for the one repo asked for, or every repo via *fed*."""+ if fed is not None:+ return await fed.run(fn)+ ctx = await _resolve_repo_context(repo)+ return [(ctx, await fn(ctx))]diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/serve.py b/packages/server/src/repowise/server/mcp_server/tool_search/serve.pynew file mode 100644index 000000000..ce05b5250--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/serve.py@@ -0,0 +1,275 @@+"""Served-row shaping: paths, freshness, slimming and freshness claims."""++from __future__ import annotations++from collections.abc import Container++from sqlalchemy import select++from repowise.core.generation.structural_labels import is_structural_title+from repowise.core.persistence.models import (+ GitMetadata,+ Page,+)++# Patchable names are read off the package at call time so patches reach them.+from repowise.server.mcp_server import tool_search as _pkg+from repowise.server.mcp_server._budget import (+ OmissionCollector,+ register_post_enforce,+ register_post_shed,+)+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT+from repowise.server.mcp_server._page_paths import (+ PAGELESS_FILE,+ add_row_paths,+ hit_file_path,+)+from repowise.server.mcp_server._query_shape import (+ _embedded_identifiers,+ _looks_like_exact_token,+)+from repowise.server.mcp_server._references import path_identity, symbol_identity++# Freshness tie-breaker, added (not multiplied) to a hit's fused relevance.+# The RRF-fused score spaces adjacent ranks ~0.05 apart, so this stays under+# one rank step: recency orders otherwise-comparable hits without overriding+# retrieval relevance. Recency itself scales it (0.5 for 90-day activity, 1.0+# for 30-day).+_FRESHNESS_TIEBREAK = 0.03+++def _attach_paths(output: list[dict], page_info: dict) -> None:+ """Stamp canonical file paths and split symbol ids into their own field.++ A ``symbol_spotlight``'s target_path is ``file.py::Symbol``: a page id, and+ not a file path. Preserve it as ``symbol_id`` for ``get_symbol`` and expose+ only the file portion through ``target_path`` / ``file``. Every search+ branch calls this one function so they agree on the row's location.+ """+ for item in output:+ item["target_path"] = page_info.get(item["page_id"], item.get("target_path", ""))+ if "::" in item["target_path"]:+ item["symbol_id"] = symbol_identity(item["target_path"])+ item["target_path"] = path_identity(item["target_path"].split("::", 1)[0])+ item["file"] = item["target_path"]+++def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:+ """Strip ``page_id`` from every result that can rebuild it, in place.++ A page id is ``compute_page_id(page_type, target_path)`` — literally+ ``f"{page_type}:{target_path}"`` (``core/generation/models.py``) — and both+ halves ship in the same row, so the field restates two of its siblings. It+ is ranking plumbing: internal uses key the fused dict on it, and none of+ that needs to reach the wire.++ Conditional rather than unconditional, and the condition is the derivation+ itself. ``_attach_paths`` writes ``target_path: ""`` for a hit whose Page+ row did not load, and a row like that cannot be rebuilt — so it keeps its+ id instead of losing one. The rule is "drop it only where it is provably+ redundant", which is also the check: if this ever stops firing, the+ invariant it rests on has changed.++ Consumers rebuild with the same expression (``ui/src/chat/source-citations``).++ ``add_row_paths`` runs first and deletes ``target_path`` wherever it equals+ the new ``path``, so the id's second half is ``target_path`` when present+ and ``path`` otherwise. Without that fallback an ordinary file row rebuilt+ as ``"file_page:"`` and kept its id.+ """+ for item in results:+ # A symbol page's id ends in ``::Symbol`` and ``_attach_paths`` moved that+ # half into ``symbol_id``, so ``target_path`` is only its file.+ target = (+ item.get("symbol_id")+ if item.get("page_type") == "symbol_spotlight"+ else None+ ) or item.get("target_path") or item.get("path", "")+ derived = f"{item.get('page_type', '')}:{target}"+ # A row for a file with no page has an id that names no page.+ if item.get("page_id") == derived or item.get("page_type") == PAGELESS_FILE:+ item.pop("page_id", None)+ return results+++def _page_language(ctx) -> str:+ """The language *ctx*'s structural page titles were generated in."""+ try:+ from repowise.core.repo_config import load_repo_config++ return str(load_repo_config(ctx.path).get("language") or "en")+ except Exception:+ return "en"+++def _slim_served_rows(+ rows: list[dict], languages: dict[str | None, str] | None = None+) -> list[dict]:+ """The last step before a search reply leaves: one location per row.++ ``path`` (plus ``symbol_id`` on symbol rows) is the row's only location, so+ ``file``, a ``page_id`` that rebuilds from them, and a ``title`` that is just+ the structural label wrapped around them are all dropped. Each is dropped+ only where the caller can provably rebuild it. ``sources: ["fts"]`` goes when+ the reply's own ``_meta.semantic_search`` already says retrieval is+ full-text-only; an index with embeddings is unchanged.++ *languages* maps a row's ``repo`` alias (``None`` for a single-repo reply) to+ the language its titles were generated in.+ """+ add_row_paths(rows)+ _drop_derivable_page_ids(rows)+ fts_only = _pkg.semantic_search_state() is False+ for row in rows:+ target = row.get("symbol_id") or row.get("path")+ if target and is_structural_title(+ (languages or {}).get(row.get("repo"), "en"),+ row.get("page_type", ""),+ target,+ row.get("title"),+ ):+ row.pop("title", None)+ if fts_only and row.get("sources") == ["fts"]:+ row.pop("sources", None)+ return rows+++def _serve_snippets(results: list[dict]) -> None:+ """Drop each snippet's leading page heading, after ranking has read it."""+ for item in results:+ if item.get("snippet"):+ item["snippet"] = _pkg.strip_leading_headings(item["snippet"])+++def _drop_internal_ranking_fields(results: list[dict]) -> None:+ """Keep calibration diagnostics internal to the ranking pipeline."""+ for item in results:+ item.pop("_coverage", None)+ item.pop("_coverage_multiplier", None)+ item.pop("_confidence_score_factor", None)+ item.pop("_raw_score", None)+++async def _load_page_info(+ session, output: list[dict], *, with_git: bool = False+) -> tuple[dict, set, dict]:+ """Batch-load target paths, tombstones, and git.++ Returns ``(page_info, tombstoned, git_map)`` where ``page_info`` maps+ page_id -> target_path, ``tombstoned`` is the set of tombstoned page_ids,+ and ``git_map`` maps file_path -> GitMetadata (empty unless ``with_git``).+ """+ page_ids = [item["page_id"] for item in output]+ res = await session.execute(+ select(Page.id, Page.target_path, Page.freshness_status).where(Page.id.in_(page_ids))+ )+ rows = res.all()+ page_info = {row[0]: row[1] for row in rows}+ tombstoned = {row[0] for row in rows if row[2] == "tombstone"}++ git_map: dict[str, GitMetadata] = {}+ if with_git:+ target_paths = [tp for tp in page_info.values() if tp]+ if target_paths:+ git_res = await session.execute(+ select(GitMetadata).where(GitMetadata.file_path.in_(target_paths))+ )+ git_map = {g.file_path: g for g in git_res.scalars().all()}+ return page_info, tombstoned, git_map+++def _apply_freshness_boost(item: dict, gm: GitMetadata | None) -> None:+ """Nudge a recently-active file up as a bounded tie-breaker (in place).++ Additive, not multiplicative: the fused relevance score is RRF-based, which+ compresses adjacent ranks to roughly ``_FRESHNESS_TIEBREAK`` apart. A+ percentage boost on that scale would let a fresh rank-2 hit leapfrog a+ stale rank-0 hit, overriding retrieval relevance with recency. Sized below+ one rank step, freshness only orders hits retrieval already ranks close+ together.+ """+ if not gm or not item.get("relevance_score"):+ return+ c30 = gm.commit_count_30d or 0+ c90 = gm.commit_count_90d or 0+ if c30 > 0:+ recency = 1.0+ elif c90 > 0:+ recency = 0.5+ else:+ recency = 0.0+ item["relevance_score"] = round(item["relevance_score"] + _FRESHNESS_TIEBREAK * recency, 4)+++def _assign_confidence(output: list[dict], score_key: str, target_key: str) -> None:+ """Derive ``target_key`` for each item from its ``score_key`` position (in place)."""+ if not output:+ return+ max_score = max((item.get(score_key) or 0) for item in output)+ for item in output:+ raw = item.get(score_key) or 0+ item[target_key] = round(raw / max_score, 2) if max_score > 0 else 0.0+++def _refresh_served_targets(response: dict, _collector: OmissionCollector | None) -> None:+ """Re-derive target-scoped freshness from the hits that survived shedding.++ ``_meta.targets`` claims which files the response served. Shedding decides+ that, so this has to run after it or the claim names rows the caller never+ received.+ """+ if response.get("truncated") and isinstance(response.get("_meta"), dict):+ response["_meta"]["targets"] = _result_paths(response.get("results") or [])+++def _refresh_served_targets_final(response: dict) -> None:+ """Same claim, re-derived after the final size guard may have cut again."""+ _refresh_served_targets(response, None)+++register_post_shed("search_codebase", _refresh_served_targets)+register_post_enforce("search_codebase", _refresh_served_targets_final)+++def _result_paths(results: list[dict]) -> list[str]:+ """File paths a result set serves, for target-scoped freshness.++ Symbol hits carry ``file``; page hits are resolved through their page type,+ so a module page's group key and an onboarding slot resolve to nothing+ rather than being handed on as if they were files. Freshness is per-file+ git metadata, and a directory has none, so a page id reaching here could+ only ever fail to match.++ An empty result set returns ``[]`` (meaning "no file content served",+ which suppresses the repo-level stale warning by design).+ """+ paths: list[str] = []+ for item in results:+ p = hit_file_path(item)+ if p:+ paths.append(p)+ return paths+++def _grep_hint_for(query: str, names: Container[str] | None = None) -> str | None:+ """Zero-result recovery hint for identifier-shaped queries, else ``None``.++ Only attached when the search produced nothing (see call sites) — a+ populated result set needs no escape hatch. Points back into the tool+ surface first; Grep is named only for the one job it genuinely wins:+ an exhaustive literal sweep of every usage (e.g. before a rename).+ """+ if _looks_like_exact_token(query):+ return (+ f"No indexed match for identifier {query!r}. Retry with "+ 'mode="symbol" (or check spelling/casing). ' + EXHAUSTIVE_SWEEP_HINT+ )+ if idents := _embedded_identifiers(query, names):+ shown = ", ".join(repr(t) for t in idents[:3])+ return (+ f"Query names identifier(s) {shown} but nothing matched. Search "+ 'the identifier alone with mode="symbol", then pipe the hit '+ "into get_symbol for its body. " + EXHAUSTIVE_SWEEP_HINT+ )+ return Nonediff --git a/packages/server/src/repowise/server/mcp_server/tool_search/structured.py b/packages/server/src/repowise/server/mcp_server/tool_search/structured.pynew file mode 100644index 000000000..c17b8b148--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/structured.py@@ -0,0 +1,395 @@+"""Symbol, path and hybrid search across one repo or a federation."""++from __future__ import annotations++from collections.abc import Container+from dataclasses import dataclass+from typing import Any++from repowise.core.persistence.database import get_session++# Patchable names are read off the package at call time so patches reach them.+from repowise.server.mcp_server import tool_search as _pkg+from repowise.server.mcp_server._federation import Federation, merge_ranked+from repowise.server.mcp_server._helpers import _get_repo+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT+from repowise.server.mcp_server._meta import build_meta as _build_meta+from repowise.server.mcp_server._page_paths import (+ file_candidates,+ hit_file_path,+)+from repowise.server.mcp_server._query_shape import (+ NOT_THE_NAMED_SYMBOL,+ _canonical_symbol_query,+ _embedded_identifiers,+ _has_exact_symbol,+ _identifier_candidates,+ _looks_like_code_name,+ _mark_not_the_named_symbol,+ _names_a_path,+)+from repowise.server.mcp_server._query_terms import content_terms+from repowise.server.mcp_server.tool_search_symbols import (+ _MAX_CANDIDATES,+ IssueFiles,+ file_path_index,+ search_paths_single,+)++from .ordering import (+ _FILE_WINDOW_OVERFETCH,+ _collapse_by_file,+ _hybrid_window,+ _protect_exact_paths,+ _protect_exact_symbols,+ _protect_named_symbols,+)+from .retrieval import _neg_score, _per_repo, _search_single_repo+from .serve import _page_language, _result_paths, _serve_snippets, _slim_served_rows+++def _lead_candidates(response: dict, issue: IssueFiles | None, limit: int) -> None:+ """Trace files lead ``candidates``. A named identifier's defining file only+ fills a free slot after the ranked ones: ranked above them, it displaces+ better hits on ordinary questions. Capped at ``limit``."""+ if issue is None or not (issue.traced or issue.named):+ return+ ranked = [c["path"] for c in response.get("candidates") or []]+ paths = list(dict.fromkeys(issue.traced + ranked + issue.named))+ response["candidates"] = [{"path": p} for p in paths[:limit]]+++def _tag_repo(items: list[dict], ctx, multi: bool) -> None:+ if multi:+ for item in items:+ item["repo"] = ctx.alias+++def _missing_named_symbols(+ candidates: list[str], symbols: list[dict], canonical: bool, concepts: list[dict]+) -> list[str]:+ """Code-shaped names the query asks after that no returned symbol matches+ exactly (or returned module path carries): they do not exist here. Judged+ per name, so an indexed name beside a missing one hides nothing. Their+ fuzzy neighbours would stand in for them, so the caller keeps only exact+ symbols; the pages are marked here as related to the question, not the+ symbol."""+ if canonical:+ return []+ page_paths = [c.get("target_path") or "" for c in concepts]+ missing = [+ c+ for c in candidates+ if _looks_like_code_name(c)+ and not _has_exact_symbol([c], symbols)+ and not _names_a_path(c, page_paths)+ ]+ if missing:+ _mark_not_the_named_symbol(concepts)+ return missing+++@dataclass(frozen=True)+class _SearchPlan:+ """One structured search: the caller's arguments plus the derived query."""++ query: str+ limit: int+ page_type: str | None+ kind: str | None+ symbol_kind: str | None+ mode: str+ names: Container[str] | None+ canonical_symbol: tuple[Any, str] | None+ symbol_query: str+ fetch: int+ multi: bool+++@dataclass+class _Pools:+ """Each row kind merged and ranked across the repos searched."""++ contexts: list[Any]+ symbols: list[dict]+ spare: list[dict]+ files: list[dict]+ indexed_paths: list[str]+ concepts: list[dict]+++def _symbol_query_for(+ query: str, mode: str, names: Container[str] | None, canonical_symbol: Any+) -> str:+ """The text the symbol scorer ranks on."""+ # Scored on the identifiers, not the prose: words like "is" would match+ # ``is_ci`` and bury the symbol the question names.+ symbol_query = query+ if canonical_symbol:+ symbol_query = canonical_symbol[1]+ if mode == "hybrid":+ _idents = _embedded_identifiers(query, names)+ # No identifier: content words, else the raw text.+ symbol_query = (+ " ".join(_idents) if _idents else " ".join(content_terms(query)) or query+ )+ return symbol_query+++def _symbol_fetch(mode: str, limit: int) -> int:+ """How many symbol rows each repo fetches."""+ # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every+ # candidate so ``fuzzy_omitted`` can count what an exact hit hides.+ fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit+ if mode == "symbol":+ fetch = _MAX_CANDIDATES+ return fetch+++async def _search_one_repo(+ ctx: Any, plan: _SearchPlan+) -> tuple[list[dict], list[dict], list[str], list[dict]]:+ """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""+ s: list[dict] = []+ f: list[dict] = []+ paths: list[str] = []+ c: list[dict] = []+ if plan.mode in ("symbol", "hybrid"):+ s = await _pkg.search_symbols_single(+ ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind+ )+ if plan.mode == "path":+ f = await search_paths_single(ctx, plan.query, plan.limit)+ async with get_session(ctx.session_factory) as session:+ paths = (await file_path_index(session, (await _get_repo(session)).id)).paths+ if plan.mode == "hybrid":+ c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)+ for item in c:+ item["type"] = "page"+ await attach_hit_symbols(ctx, plan.query, c)+ for rows in (s, f, c):+ _tag_repo(rows, ctx, plan.multi)+ return s, f, paths, c+++def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:+ """Merge each repo's rows into one ranked pool per row kind."""+ parts = [part for _, part in answered]+ return _Pools(+ contexts=[ctx for ctx, _ in answered],+ symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),+ # Rows past each repo's limit only backfill slots that collapsing frees.+ spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),+ files=merge_ranked((p[1] for p in parts), _neg_score("score")),+ indexed_paths=[path for p in parts for path in p[2]],+ concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),+ )+++def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:+ """Keep exact name and path matches ahead of their fuzzy neighbours."""+ if plan.mode == "symbol":+ pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)+ elif plan.mode == "hybrid" and candidates:+ pools.symbols = _protect_named_symbols(candidates, pools.symbols)+ if plan.mode == "path":+ pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)+++def _symbol_mode_results(+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool+) -> tuple[list[dict], int]:+ """Symbol mode rows, and how many fuzzy names an exact hit hides."""+ # An exact hit is the answer; its fuzzy neighbours are only counted.+ fuzzy_omitted = 0+ pool = pools.symbols + pools.spare+ named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []+ if named:+ canonical_symbol = plan.canonical_symbol+ ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()+ shown = {id(s) for s in named}+ fuzzy_omitted = sum(+ 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()+ )+ return (named or pools.symbols)[: plan.limit], fuzzy_omitted+++def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:+ """Hybrid rows: symbol matches interleaved with concept pages for new files."""+ window = _hybrid_window(+ plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names+ )+ # One row per file; freed slots fill with the next pages, then symbols.+ served = {id(item) for item in window}+ rest = [+ item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served+ ]+ results = _collapse_by_file(window + rest)[: plan.limit]+ # Non-file pages leave the window unrefilled unless the caller asked for+ # pages by type or kind="doc".+ if not plan.page_type and plan.kind != "doc":+ results = [item for item in results if hit_file_path(item)]+ return results+++def _select_results(+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool+) -> tuple[list[dict], int]:+ """The served rows for the plan's mode, and the symbol-mode fuzzy count."""+ if plan.mode == "symbol":+ return _symbol_mode_results(plan, pools, candidates, exact)+ if plan.mode == "path":+ return pools.files[: plan.limit], 0+ return _hybrid_mode_results(plan, pools, exact), 0+++async def _single_repository(contexts: list[Any], multi: bool) -> Any:+ """The repository row a single-repo search ran against, else None."""+ if multi or not contexts:+ return None+ async with get_session(contexts[0].session_factory) as session:+ return await _get_repo(session)+++def _exact_match_note(+ candidates: list[str],+ exact: bool,+ missing: list[str],+ fuzzy_omitted: int,+ has_results: bool,+ grep_hint: str | None,+) -> str | None:+ """The note qualifying an identifier query's results, if one is owed."""+ if missing:+ shown = ", ".join(repr(c) for c in missing[:3])+ # An empty window's grep_hint already carries the sweep advice.+ pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""+ sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT+ return (+ f"No indexed symbol is named {shown}, so no symbol is returned "+ f"for it.{pages} Recheck the spelling, or search a shorter part "+ f"of the name.{sweep}"+ )+ if not exact:+ shown = ", ".join(repr(c) for c in candidates[:3])+ return (+ f"No indexed symbol exactly matches {shown}. The results are "+ "fuzzy neighbours ranked by token overlap — confirm a hit names "+ "what you meant before relying on it. If you expected an exact "+ "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT+ )+ if fuzzy_omitted:+ return (+ f"{fuzzy_omitted} other symbols contain this name; search a "+ "longer or partial name to list them."+ )+ return None+++def _match_named(+ plan: _SearchPlan, pools: _Pools, candidates: list[str]+) -> tuple[bool, list[str]]:+ """Whether a returned symbol names the query exactly, and the names none does.++ Missing names leave only exact symbols in *pools*.+ """+ exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False+ missing = _missing_named_symbols(+ candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts+ )+ if missing:+ pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]+ pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]+ return exact, missing+++def _attach_match_notes(+ response: dict,+ candidates: list[str],+ exact: bool,+ missing: list[str],+ grep_hint: str | None,+) -> None:+ """Exact-match flag, its note, and the grep fallback for an empty window."""+ results = response["results"]+ # Fuzzy neighbours of a missing name look authoritative, so say so.+ if candidates:+ response["exact_match"] = exact and not missing+ note = _exact_match_note(+ candidates,+ exact,+ missing,+ response.get("fuzzy_omitted", 0),+ bool(results),+ grep_hint,+ )+ if note:+ response["note"] = note+ if grep_hint and not results:+ response["grep_hint"] = grep_hint+++async def _structured_search(+ query: str,+ limit: int,+ page_type: str | None,+ kind: str | None,+ symbol_kind: str | None,+ repo: str | None,+ mode: str,+ grep_hint: str | None,+ names: Container[str] | None = None,+ fed: Federation | None = None,+) -> dict:+ """Run symbol / path / hybrid search, across repos when federated, and shape the response.++ ``grep_hint`` is attached only when nothing was found.+ """+ multi = fed is not None and fed.multi+ canonical_symbol = _canonical_symbol_query(query)+ plan = _SearchPlan(+ query=query,+ limit=limit,+ page_type=page_type,+ kind=kind,+ symbol_kind=symbol_kind,+ mode=mode,+ names=names,+ canonical_symbol=canonical_symbol,+ symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),+ fetch=_symbol_fetch(mode, limit),+ multi=multi,+ )++ async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:+ return await _search_one_repo(ctx, plan)++ pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)+ candidates = _identifier_candidates(query, mode, names)+ _protect_exact(plan, pools, candidates)+ exact, missing = _match_named(plan, pools, candidates)++ results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)+ repository = await _single_repository(pools.contexts, multi)++ response: dict = {+ "results": results,+ "mode": mode,+ "_meta": _build_meta(repository=repository, targets=_result_paths(results)),+ }+ if fuzzy_omitted:+ response["fuzzy_omitted"] = fuzzy_omitted+ # Not ``candidates``: that name holds the query's identifiers the note quotes.+ if file_cands := file_candidates(results, limit=limit):+ response["candidates"] = file_cands+ _attach_match_notes(response, candidates, exact, missing, grep_hint)+ if fed is not None:+ fed.annotate(response, results)+ # Last: slims fields the steps above still read.+ _slim_served_rows(+ results,+ {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},+ )+ _serve_snippets(results)+ return responsediff --git a/packages/server/src/repowise/server/mcp_server/tool_search/tool.py b/packages/server/src/repowise/server/mcp_server/tool_search/tool.pynew file mode 100644index 000000000..2843b758e--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/tool_search/tool.py@@ -0,0 +1,282 @@+"""MCP Tool 5: search_codebase — semantic search over the wiki."""++from __future__ import annotations++import logging+from typing import Any++from repowise.core.persistence.database import get_session+from repowise.core.registry import ToolRecipe+from repowise.core.registry import mcp_tool_registry as mcp++# Patchable names are read off the package at call time so patches reach them.+from repowise.server.mcp_server import tool_search as _pkg+from repowise.server.mcp_server._federation import Federation+from repowise.server.mcp_server._helpers import (+ _get_repo,+ _resolve_repo_context,+ attach_ignored_arguments,+ filter_dicts_by_key,+ resolve_enum_argument,+)+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols+from repowise.server.mcp_server._line_hits import attach_line_hits+from repowise.server.mcp_server._meta import build_meta as _build_meta+from repowise.server.mcp_server._page_paths import file_candidates+from repowise.server.mcp_server._query_shape import (+ _VALID_MODES,+ _fetch_limit_for,+ _resolve_mode,+)+from repowise.server.mcp_server._retrieval_rank import (+ boost_named_paths,+ rerank_pages_first,+)+from repowise.server.mcp_server.tool_search_symbols import (+ IssueFiles,+ file_path_index,+ indexed_names,+ issue_files,+)++from .ordering import _collapse_by_file+from .ranking import (+ _VALID_KINDS,+ _dedup_decisions,+ _downweight_decisions,+ _downweight_test_pages,+ _filter_by_kind,+ _rescue_all_decision_window,+ _reserve_symbol_tail,+ _sort_demoting_noise,+)+from .retrieval import _federated_search, _fused_retrieve, _per_repo+from .serve import (+ _apply_freshness_boost,+ _assign_confidence,+ _attach_paths,+ _drop_internal_ranking_fields,+ _grep_hint_for,+ _load_page_info,+ _page_language,+ _result_paths,+ _serve_snippets,+ _slim_served_rows,+)+from .structured import _structured_search++_log = logging.getLogger("repowise.mcp.search")+++def _query_from_pattern(query: str, pattern: str | None, ignored: list[dict[str, Any]]) -> str:+ """``pattern`` fills an empty ``query``; beside a real one it is reported ignored."""+ # Hosts that defer tool schemas let a model guess grep's argument name.+ if pattern is not None and not query.strip():+ return pattern+ if pattern is not None:+ ignored.append(+ {"argument": "pattern", "values": [pattern], "valid": [], "superseded_by": "query"}+ )+ return query+++def _resolve_kind_and_mode(+ kind: str | None, mode: str, ignored: list[dict[str, Any]]+) -> tuple[str | None, str]:+ """Validate ``kind`` and ``mode``; an unknown value is dropped and reported."""+ kind = resolve_enum_argument(kind, _VALID_KINDS, argument="kind", ignored=ignored)+ # Modes are case-insensitive; only a genuine miss is dropped, under its own spelling.+ if mode is not None and mode.lower() in _VALID_MODES:+ mode = mode.lower()+ mode = resolve_enum_argument(mode, _VALID_MODES, argument="mode", ignored=ignored)+ return kind, mode+++async def _indexed_names(repo: str | None, fed: Federation | None, query: str) -> set[str]:+ """Indexed names the query mentions, loaded once so every stage validates alike."""+ names: set[str] = set()+ for _, repo_names in await _per_repo(repo, fed, lambda ctx: indexed_names([ctx], query)):+ names |= repo_names+ return names+++async def _issue_files(repo: str | None, query: str, names: set[str]) -> IssueFiles | None:+ """Files the query traces or names; single repo only (a federation has no one tree)."""+ if repo == "all":+ return None+ try:+ return await issue_files(await _resolve_repo_context(repo), query, names)+ except Exception:+ _log.warning("search_codebase: issue file lookup failed", exc_info=True)+ return None+++async def _rank_concept_pages(ctx: Any, output: list[dict], query: str, repo_id: Any) -> list[dict]:+ """Drop tombstoned and excluded pages, then boost, demote and dedupe the rest."""+ async with get_session(ctx.session_factory) as session:+ page_info, tombstoned, git_map = await _load_page_info(session, output, with_git=True)+ path_index = await file_path_index(session, repo_id)+ # Tombstoned pages document deleted/renamed files — never results.+ output = [item for item in output if item["page_id"] not in tombstoned]+ _attach_paths(output, page_info)+ output = filter_dicts_by_key(output, "target_path", _pkg._get_exclude_spec(ctx.path))+ for item in output:+ target_path = page_info.get(item["page_id"])+ _apply_freshness_boost(item, git_map.get(target_path) if target_path else None)+ output = rerank_pages_first(output, query, score_key="relevance_score", floor=0.5)+ # After the coverage rerank, whose window-relative weights cannot tell+ # a word rare across the repo from one rare in these few hits.+ boost_named_paths(+ output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"+ )+ _downweight_test_pages(output, query)+ _sort_demoting_noise(output, query)+ return _dedup_decisions(output)+++async def _concept_search(+ query: str,+ limit: int,+ page_type: str | None,+ kind: str | None,+ repo: str | None,+ issue: IssueFiles | None,+ grep_hint: str | None,+) -> tuple[dict, Any]:+ """Single-repo concept search; returns the response and the repo context it read."""+ ctx = await _resolve_repo_context(repo)+ async with get_session(ctx.session_factory) as session:+ repository = await _get_repo(session)+ await _pkg._wait_for_vector_store(ctx)++ # Over-fetch: post-filters trim hits, and decision down-weighting needs file+ # pages inside the window to promote.+ fetch_limit = _fetch_limit_for(limit, kind)+ output = await _fused_retrieve(ctx, query, fetch_limit, page_type)+ _downweight_decisions(output, query)+ output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)+ if output:+ output = await _rank_concept_pages(ctx, output, query, repository.id)++ output = _filter_by_kind(output, kind)+ # A file page and a symbol page of one file are one place to look.+ output = _collapse_by_file(output)+ # Files the page retrievers cannot see (a private helper, a local name) get+ # the weakest tail slots.+ output = _reserve_symbol_tail(output, limit, kind)+ _drop_internal_ranking_fields(output)+ # The full ranked pool, kept so ``candidates`` can reach past the cut.+ ranked = list(output)+ output = output[:limit]+ await attach_hit_symbols(ctx, query, output)+ _assign_confidence(output, "relevance_score", "confidence_score")++ response: dict = {+ "results": output,+ "_meta": _build_meta(repository=repository, targets=_result_paths(output)),+ }+ if candidates := file_candidates(ranked, limit=limit):+ response["candidates"] = candidates+ _pkg._lead_candidates(response, issue, limit)+ if grep_hint and not output:+ response["grep_hint"] = grep_hint+ return response, ctx+++ surface_order=40,+ artifact_type="search_results",+ presentation="search_results",+ evidence_basis="inferred",+ recipes=(+ ToolRecipe(+ "find_raw_hits",+ 'search_codebase(query="identifier or path", mode="auto")',+ ("search_codebase",),+ ),+ ),+)+async def search_codebase(+ query: str = "",+ limit: int = 5,+ page_type: str | None = None,+ kind: str | None = None,+ repo: str | None = None,+ mode: str = "auto",+ symbol_kind: str | None = None,+ pattern: str | None = None,+) -> dict:+ """Find code by concept, symbol, or path — hybrid codebase search.++ For questions (how/where/why), call get_answer instead: it runs this+ retrieval internally and returns a cited answer. Use this for raw+ ranked hits: enumerating matches, resolving an identifier to a symbol_id,+ or scoping get_context.++ mode="auto" routes by query shape: identifiers search symbols (symbol_id,+ path, line bounds for get_symbol), paths resolve files (for get_context),+ prose runs wiki-semantic search, and mixed queries run hybrid (symbol hits,+ then file-backed pages only). Decision records rank below file pages+ unless the query is why-shaped.++ Rows naming a file carry `path`; concept pages add `symbols`+ (name:line) the query matches; the top 3 add `matched_lines`+ (line, text), sample lines sharing its words. `candidates` lists up to `limit`+ distinct files to Read, best first. Identifier or literal queries also+ return `lines` (path, line, kind, text; definitions first) read from live+ files; when `complete` is true they are every match, so no grep is needed.++ Args:+ query: identifier, path, or natural language.+ limit: max results (default 5); distinct files outside mode="symbol"+ (same-file symbols in `symbols`).+ page_type: file_page (per-file docs) or module_page (subsystem+ pages); any stored page type filters.+ kind: implementation | test | config | doc (concept/symbol modes).+ repo: alias, or "all" for workspace-wide.+ mode: auto | concept | symbol | path | hybrid.+ symbol_kind: filter symbol hits (function|class|method|...).+ pattern: alias for query.+ """+ ignored: list[dict[str, Any]] = []+ query = _query_from_pattern(query, pattern, ignored)+ if not query.strip():+ return {+ "results": [],+ "error": "search_codebase requires `query` (or its alias `pattern`).",+ "_meta": _build_meta(),+ }+ kind, mode = _resolve_kind_and_mode(kind, mode, ignored)+ # One fan-out state for the whole call: a repo that fails one stage is+ # skipped by the next and reported once.+ fed = Federation() if repo == "all" else None+ names = await _indexed_names(repo, fed, query)+ grep_hint = _grep_hint_for(query, names)+ resolved_mode = _resolve_mode(query, mode, names)+ issue = await _issue_files(repo, query, names)++ if resolved_mode in ("symbol", "path", "hybrid"):+ structured = await _structured_search(+ query, limit, page_type, kind, symbol_kind, repo, resolved_mode, grep_hint, names, fed+ )+ _pkg._lead_candidates(structured, issue, limit)+ await attach_line_hits(structured, query, resolved_mode, names, repo)+ attach_ignored_arguments(structured, ignored)+ return structured++ if fed is not None:+ # kind is filtered per repo before each repo's limit cut, so the fused+ # list is already kind-pure and full.+ federated = await _federated_search(query, limit, page_type, kind, fed)+ if grep_hint and not federated.get("results"):+ federated["grep_hint"] = grep_hint+ attach_ignored_arguments(federated, ignored)+ return federated++ response, ctx = await _concept_search(query, limit, page_type, kind, repo, issue, grep_hint)+ await attach_line_hits(response, query, resolved_mode, names, repo)+ attach_ignored_arguments(response, ignored)+ # Last, so nothing above has to know the field is on its way out.+ _slim_served_rows(response["results"], {None: _page_language(ctx)})+ _serve_snippets(response["results"])+ return responsediff --git a/tests/unit/server/mcp/test_vector_search_timeout.py b/tests/unit/server/mcp/test_vector_search_timeout.pyindex c29650b2d..151f36b96 100644--- a/tests/unit/server/mcp/test_vector_search_timeout.py+++ b/tests/unit/server/mcp/test_vector_search_timeout.py@@ -98,7 +98,10 @@ def test_no_hardcoded_eight_second_vector_budget() -> None:"""Pins the fix: every vector site reads the shared budget."""from pathlib import Path- root = Path(tool_search.__file__).parent- for path in (root / "tool_search.py", *sorted((root / "_answer_pipeline").glob("*.py"))):+ root = Path(tool_search.__file__).parent.parent+ for path in (+ *sorted((root / "tool_search").glob("*.py")),+ *sorted((root / "_answer_pipeline").glob("*.py")),+ ):text = path.read_text(encoding="utf-8")assert "timeout=8.0" not in text, f"{path.name} still hardcodes the old vector budget"