Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(mcp): split tool_search into a package of search stages (#4044)

Raghav Chamadiya7823cd1de1c1
diff --git a/docs/LINEAGE.md b/docs/LINEAGE.md
index 47f571e0e..aa3271f58 100644
--- a/docs/LINEAGE.md
+++ b/docs/LINEAGE.md
@@ -127,7 +127,7 @@ and infers the parameters and return values a behaviour-preserving signature nee
| Year | Work | What we take from it | Where it lives |
|---|---|---|---|
| 2009 | Robertson and Zaragoza, *The probabilistic relevance framework: BM25 and beyond*, Foundations and Trends in IR | BM25 ranking over full-text search, with per-column weights | `persistence/search.py` |
-| 2009 | Cormack, Clarke and Büttcher, *Reciprocal rank fusion outperforms Condorcet and individual rank learning methods*, SIGIR | Merging full-text and semantic results | `packages/server/src/repowise/server/mcp_server/tool_search.py` |
+| 2009 | Cormack, Clarke and Büttcher, *Reciprocal rank fusion outperforms Condorcet and individual rank learning methods*, SIGIR | Merging full-text and semantic results | `packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py` |
**How we checked it.** On 42 sealed retrieval tasks, held out from every round of
tuning and graded without a model, `get_answer` found 0.876 of the files a fix needed,
diff --git a/docs/reference/COMPUTED_GLOSSARY.md b/docs/reference/COMPUTED_GLOSSARY.md
index 6fb15c7d1..9babb562f 100644
--- a/docs/reference/COMPUTED_GLOSSARY.md
+++ b/docs/reference/COMPUTED_GLOSSARY.md
@@ -263,8 +263,8 @@ workspace overlays, MCP responses, and CLI output.
| Answer payload | Cached `get_answer` result. | `get_answer()` | `{answer, citations, confidence, fallback_targets, retrieval}` |
| Retrieval hit | Search hit hydrated with page metadata and summary. | `tool_answer.py` retrieval pipeline | `{target_path: "src/auth.py", score: 3.2, summary: "..."}` |
| Retrieval dominance | Gating logic comparing top and second search scores. | `tool_answer.py` | Top score high enough to answer from dominant hit |
-| Federated RRF score | Reciprocal rank fusion score for workspace search across repos. | `tool_search.py` | `rrf_score: 0.0164` |
-| Confidence score | Normalized workspace search confidence. | `tool_search.py` | `confidence_score: 0.87` |
+| Federated RRF score | Reciprocal rank fusion score for workspace search across repos. | `tool_search/retrieval.py` | `rrf_score: 0.0164` |
+| Confidence score | Normalized workspace search confidence. | `tool_search/retrieval.py` | `confidence_score: 0.87` |
## Persistence Tables And Stored Entities
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.py
deleted file mode 100644
index 791895736..000000000
--- a/packages/server/src/repowise/server/mcp_server/tool_search.py
+++ /dev/null
@@ -1,1558 +0,0 @@
-"""MCP Tool 5: search_codebase — semantic search over the wiki."""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import logging
-import re
-from collections.abc import Container, Sequence
-from dataclasses import dataclass
-from typing import Any
-
-from sqlalchemy import select
-
-from repowise.core.generation.structural_labels import is_structural_title
-from repowise.core.persistence.database import get_session
-from repowise.core.persistence.models import (
- GitMetadata,
- Page,
-)
-from repowise.core.persistence.search import strip_leading_headings
-from repowise.core.providers.embedding import store_has_semantic_vectors
-from repowise.core.registry import ToolRecipe
-from repowise.core.registry import mcp_tool_registry as mcp
-from repowise.core.test_paths import is_test_path, is_test_related_path
-from repowise.server.mcp_server._answer_pipeline import (
- _FILENAME_LEG_RRF_K,
- _RRF_K,
- _RRF_SCORE_SCALE,
- _SYMBOL_LEG_RRF_K,
- _safe_filename_search,
- _safe_symbol_search,
- leg_ranks,
-)
-from repowise.server.mcp_server._budget import (
- OmissionCollector,
- register_post_enforce,
- register_post_shed,
-)
-from repowise.server.mcp_server._federation import Federation, merge_ranked
-from repowise.server.mcp_server._helpers import (
- _VECTOR_TIMEOUT_ENV,
- _get_exclude_spec,
- _get_repo,
- _resolve_repo_context,
- attach_ignored_arguments,
- filter_dicts_by_key,
- resolve_enum_argument,
- vector_search_timeout_s,
-)
-from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
-from repowise.server.mcp_server._line_hits import attach_line_hits
-from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT, semantic_search_state
-from repowise.server.mcp_server._meta import build_meta as _build_meta
-from repowise.server.mcp_server._page_paths import (
- FILE_ROW_TYPES,
- PAGELESS_FILE,
- add_row_paths,
- file_candidates,
- file_path_of,
- hit_file_path,
- pageless_path,
-)
-from repowise.server.mcp_server._query_shape import (
- _DECISION_DOWNWEIGHT,
- _MIN_RELEVANCE_SCORE,
- _VALID_MODES,
- NOT_THE_NAMED_SYMBOL,
- _canonical_symbol_query,
- _embedded_identifiers,
- _fetch_limit_for,
- _has_exact_symbol,
- _identifier_candidates,
- _is_why_shaped,
- _looks_like_code_name,
- _looks_like_exact_token,
- _mark_not_the_named_symbol,
- _names_a_path,
- _qual_norm,
- _resolve_mode,
- _symbol_matches_name,
- path_tokens,
-)
-from repowise.server.mcp_server._query_terms import content_terms
-from repowise.server.mcp_server._references import path_identity, symbol_identity
-from repowise.server.mcp_server._retrieval_rank import (
- boost_named_paths,
- rerank_pages_first,
-)
-from repowise.server.mcp_server.tool_search_symbols import (
- _MAX_CANDIDATES,
- IssueFiles,
- file_path_index,
- indexed_names,
- issue_files,
- search_paths_single,
- search_symbols_single,
-)
-
-_log = logging.getLogger("repowise.mcp.search")
-
-# Freshness tie-breaker, added (not multiplied) to a hit's fused relevance.
-# The RRF-fused score spaces adjacent ranks ~0.05 apart, so this stays under
-# one rank step: recency orders otherwise-comparable hits without overriding
-# retrieval relevance. Recency itself scales it (0.5 for 90-day activity, 1.0
-# for 30-day).
-_FRESHNESS_TIEBREAK = 0.03
-
-
-def _protect_exact_symbols(query: str, symbols: list[dict]) -> list[dict]:
- """Stable-partition an exact identifier or ``path::Symbol`` hit to the head."""
- if not symbols:
- return symbols
- canonical = _canonical_symbol_query(query)
- qnorm = query.strip().lower().replace("\\", "/")
-
- def exact(item: dict) -> bool:
- symbol_id = (item.get("symbol_id") or "").lower().replace("\\", "/")
- if canonical:
- return symbol_id == qnorm
- return qnorm in {
- (item.get("name") or "").lower(),
- _qual_norm(item.get("qualified_name")),
- }
-
- protected = [item for item in symbols if exact(item)]
- return protected + [item for item in symbols if not exact(item)]
-
-
-def _protect_named_symbols(candidates: list[str], symbols: list[dict]) -> list[dict]:
- """Stable-partition symbols exactly named by identifiers embedded in prose."""
- wanted = {candidate.strip().lower() for candidate in candidates if candidate.strip()}
- wanted |= {_qual_norm(candidate) for candidate in candidates if candidate.strip()}
-
- protected = [item for item in symbols if _symbol_matches_name(item, wanted)]
- return protected + [item for item in symbols if not _symbol_matches_name(item, wanted)]
-
-
-def _protect_exact_paths(
- query: str, files: list[dict], paths: Sequence[str] = ()
-) -> list[dict]:
- """Stable-partition an exact path hit ahead of basename/fuzzy neighbours."""
- tokens = path_tokens(query, paths) or [query]
- wanted = {t.strip().lower().replace("\\", "/") for t in tokens}
- protected = [item for item in files if (item.get("file") or "").lower() in wanted]
- return protected + [item for item in files if (item.get("file") or "").lower() not in wanted]
-
-
-def _prose_dominates(query: str, identifiers: list[str]) -> bool:
- """True when natural-language tokens outnumber the identifier tokens a query
- carries: the query reads as prose that merely *mentions* a symbol, not a
- symbol lookup dressed in a few words. Drives hybrid ordering below."""
- ident_count = len(identifiers)
- if ident_count == 0:
- return False
- total = len(re.findall(r"[A-Za-z0-9_]+", query))
- return (total - ident_count) > ident_count
-
-
-def _interleave_hybrid(
- query: str,
- symbols: list[dict],
- concepts: list[dict],
- limit: int,
- exact: bool,
- names: Container[str] | None = None,
-) -> list[dict]:
- """Order one hybrid result window from the two incomparable score scales.
-
- Symbol SQL scores (~43, +100 on an exact-name hit) and concept relevance
- (0 to 1) can't be merge-sorted, so we interleave by block. Default: symbols
- lead, reserving up to half the window for concept pages so a flood of symbol
- matches can't truncate every page out.
-
- The exception is the score-scale trap. When NO returned symbol matches the
- query's identifier exactly AND the query is mostly prose, leading with fuzzy
- symbol hits buries the page the caller actually wants: the generic ``.get``
- methods that outscore the ``answer.py`` page for "how does retrieval feed
- synthesis in get_answer". There, concept pages lead and the fuzzy symbols
- fall to the tail (nothing is dropped, only reordered within the window).
- """
- if not exact and concepts and _prose_dominates(query, _embedded_identifiers(query, names)):
- reserved = min(len(symbols), limit // 2)
- return (concepts[: max(1, limit - reserved)] + symbols)[:limit]
- reserved = min(len(concepts), limit // 2)
- return (symbols[: max(1, limit - reserved)] + concepts)[:limit]
-
-
-# Code-location windows over-fetch this many times ``limit`` so collapsing
-# same-file hits still leaves ``limit`` distinct files to serve.
-_FILE_WINDOW_OVERFETCH = 2
-# Other symbol names a collapsed row carries in ``symbols``.
-_MERGED_SYMBOL_CAP = 5
-
-
-def _collapse_by_file(hits: list[dict]) -> list[dict]:
- """One row per (repo, file), best first; rows naming no file pass through.
-
- The first row of a file wins. A later symbol hit in the same file folds
- into the winner's ``symbols`` as ``name:line`` (capped) instead of taking a
- slot of its own, so ``limit`` buys distinct files; past the cap a trailing
- ``+N more`` entry counts the rest. Precedent:
- ``symbol_backed_pages`` collapses the same way for the concept tail.
- """
- first_of: dict[tuple, dict] = {}
- extra: dict[int, int] = {}
- out: list[dict] = []
- for hit in hits:
- path = hit_file_path(hit)
- if path is None:
- out.append(hit)
- continue
- key = (hit.get("repo"), path)
- first = first_of.get(key)
- if first is None:
- first_of[key] = hit
- out.append(hit)
- elif hit.get("type") == "symbol" and first.get("type") == "symbol":
- merged = first.setdefault("symbols", [])
- if len(merged) < _MERGED_SYMBOL_CAP:
- merged.append(f"{hit.get('name')}:{hit.get('start_line')}")
- else:
- extra[id(first)] = extra.get(id(first), 0) + 1
- for hit in out:
- if id(hit) in extra:
- hit["symbols"].append(f"+{extra[id(hit)]} more")
- return out
-
-
-def _hybrid_window(
- query: str,
- symbols: list[dict],
- concepts: list[dict],
- limit: int,
- exact: bool,
- names: Container[str] | None = None,
-) -> list[dict]:
- """The hybrid window, with no page for a file a shown symbol row covers.
-
- Pages are dropped only for symbols the window actually shows, then the
- window is rebuilt so the freed slots backfill. Concepts only shrink and the
- symbol share only grows, so this settles in a few passes.
- """
- while True:
- window = _interleave_hybrid(query, symbols, concepts, limit, exact, names)
- shown = {item.get("file") for item in window if item.get("type") == "symbol"}
- kept = [c for c in concepts if c.get("target_path") not in shown]
- if len(kept) == len(concepts):
- return window
- concepts = kept
-
-
-def _downweight_decisions(output: list[dict], query: str) -> None:
- """Scale decision_record relevance in place unless the query is why-shaped."""
- if _is_why_shaped(query):
- return
- for item in output:
- if item.get("page_type") == "decision_record" and item.get("relevance_score"):
- item["relevance_score"] = round(item["relevance_score"] * _DECISION_DOWNWEIGHT, 4)
-
-
-# Test file pages compete against the implementation they exercise and win
-# retrieval on any shared vocabulary ("conftest" for a fixtures question,
-# "decision" for a downweight question) — observed live as a test file ranking
-# #1 for a plain implementation query. A test is rarely a better first Read than
-# the code under test, so demote it, unless the query is explicitly about tests.
-_TEST_DOWNWEIGHT = 0.6
-
-_TEST_QUERY_RE = re.compile(
- r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",
- re.IGNORECASE,
-)
-
-
-def _is_test_query(query: str) -> bool:
- """True when the query is explicitly about tests, so test pages rank naturally."""
- return bool(_TEST_QUERY_RE.search(query))
-
-
-def _is_test_page(item: dict) -> bool:
- """True when a hit is a whole-file row for a test file.
-
- Tests only, deliberately not test support: a ``conftest.py`` or a fixture
- module is often exactly what the person searching wanted, so it keeps its
- rank. The ``kind`` filter counts both (see :func:`_classify_hit_kind`) —
- asking for tests and asking to rank tests lower are different questions.
- """
- return item.get("page_type") in FILE_ROW_TYPES and is_test_path(item.get("target_path") or "")
-
-
-def _downweight_test_pages(output: list[dict], query: str) -> None:
- """Scale test file_page relevance in place unless the query is about tests."""
- if _is_test_query(query):
- return
- for item in output:
- if item.get("relevance_score") and _is_test_page(item):
- item["relevance_score"] = round(item["relevance_score"] * _TEST_DOWNWEIGHT, 4)
-
-
-def _sort_demoting_noise(output: list[dict], query: str) -> None:
- """Sort by relevance, ranking retrieval noise below every real page.
-
- Two classes crowd the top ranks on a plain implementation query and are
- demoted absolutely for their query class (the relevance score still orders
- each class among itself):
-
- - decision records — short dense titles win cosine similarity by a margin
- wider than the multiplicative down-weight (observed live as 5/5 irrelevant
- decisions for one query). Rationale questions are get_why's territory, so
- a why-shaped query ranks them naturally instead.
- - test file pages — a test is rarely a better first Read than the code it
- exercises. A query explicitly about tests ranks them naturally instead.
- """
- why = _is_why_shaped(query)
- test_focused = _is_test_query(query)
-
- def key(item: dict) -> tuple:
- pt = item.get("page_type")
- is_decision = (not why) and pt == "decision_record"
- is_test = (not test_focused) and _is_test_page(item)
- return (1 if (is_decision or is_test) else 0, -(item.get("relevance_score") or 0.0))
-
- output.sort(key=key)
-
-
-def _norm_decision_title(title: str) -> str:
- """Normalize a decision title to collapse near-duplicate phrasings."""
- return re.sub(r"[^a-z0-9]+", " ", (title or "").lower()).strip()
-
-
-def _dedup_decisions(output: list[dict]) -> list[dict]:
- """Collapse near-duplicate decision records by normalized title (order-preserving).
-
- Live search returned five near-identical "CLI incremental update regenerates
- only affected pages" variants filling positions 3-8. Callers dedup AFTER the
- score sort, so the surviving variant is the highest-scored one; non-decision
- pages always pass through untouched.
- """
- seen: set[str] = set()
- out: list[dict] = []
- for item in output:
- if item.get("page_type") == "decision_record":
- key = _norm_decision_title(item.get("title", ""))
- if key and key in seen:
- continue
- seen.add(key)
- out.append(item)
- return out
-
-
-async def _non_decision_fallback(ctx, query: str, fetch_limit: int) -> list[dict]:
- """Wider re-fetch keeping only non-decision pages.
-
- Guard for the window-saturation failure: when the entire over-fetched
- window is decision records, demotion has nothing to promote and a
- non-why query returns zero implementation pages. Fetch 4x wider, drop
- decisions, and let the caller merge the survivors.
- """
- results = []
- src = "vector"
- # Skipped on a keyless index, which then falls through to the FTS branch.
- if store_has_semantic_vectors(ctx.vector_store):
- try:
- results = await asyncio.wait_for(
- ctx.vector_store.search(query, limit=fetch_limit * 4),
- timeout=vector_search_timeout_s(),
- )
- except TimeoutError:
- _log.warning("Vector re-fetch timed out; falling back to full-text")
- except Exception:
- _log.debug("Vector re-fetch failed; falling back to full-text", exc_info=True)
- if not results:
- src = "fts"
- with contextlib.suppress(Exception):
- results = await ctx.fts.search(query, limit=fetch_limit * 4)
-
- out = []
- for r in results:
- if r.page_type == "decision_record":
- continue
- if r.score < _MIN_RELEVANCE_SCORE:
- continue
- out.append(
- {
- "page_id": r.page_id,
- "title": r.title,
- "page_type": r.page_type,
- "snippet": r.snippet,
- "relevance_score": r.score,
- "sources": [src],
- }
- )
- return out
-
-
-async def _rescue_all_decision_window(
- ctx, output: list[dict], query: str, fetch_limit: int
-) -> list[dict]:
- """Merge non-decision pages into an all-decision result window.
-
- No-op unless the query is non-why AND every current hit is a decision
- record. Deduplicates by page_id.
- """
- if not output or _is_why_shaped(query):
- return output
- if not all(item.get("page_type") == "decision_record" for item in output):
- return output
- fallback = await _non_decision_fallback(ctx, query, fetch_limit)
- seen = {item["page_id"] for item in output}
- output.extend(item for item in fallback if item["page_id"] not in seen)
- return output
-
-
-# Slots at the tail of a concept window reserved for files reached through the
-# symbol index rather than through their page. A generated file page renders
-# only public symbols, so a question about a private helper or a local name has
-# nothing to match in either the full-text index or the embedding. The symbol
-# index is where those names live. One slot in three, taken from the weakest
-# end of the window, is a cheap price for reaching a class of file the page
-# retrievers structurally cannot see. No-op when the symbol leg finds nothing.
-_SYMBOL_TAIL_DIVISOR = 3
-
-
-def _symbol_tail_reserve(limit: int) -> int:
- """How many tail slots the symbol leg may take from a window of *limit*."""
- return limit // _SYMBOL_TAIL_DIVISOR
-
-
-def _reserve_symbol_tail(output: list[dict], limit: int, kind: str | None) -> list[dict]:
- """Give the weakest window slots to symbol-leg files ranked past ``limit``.
-
- Concept hits keep the head. Files the symbol leg reached that fusion left
- outside the window take up to :func:`_symbol_tail_reserve` slots, and the
- concept hits they displaced fall in behind them (nothing is dropped).
- """
- reserve = _symbol_tail_reserve(limit)
- if reserve <= 0 or kind in ("config", "doc"):
- # config/doc windows are asking for pages with no symbols in them.
- return output
- extra = [item for item in output[limit:] if "symbol" in item.get("sources", ())][:reserve]
- if not extra:
- return output
- head = output[: limit - len(extra)]
- taken = {id(item) for item in head + extra}
- return head + extra + [item for item in output if id(item) not in taken]
-
-
-# Path-prefix heuristics for the ``kind`` filter. We classify a hit's
-# target_path against these prefixes; if none match, the hit falls into
-# ``other`` and is dropped only when the caller asked for a specific kind.
-# Tests are not in this list: they come from ``repowise.core.test_paths``, the
-# same rules that stamped the file's ``is_test`` flag at ingestion.
-_CONFIG_PATH_TOKENS = (
- "pyproject.toml",
- "package.json",
- "tsconfig",
- "setup.py",
- "setup.cfg",
- "/.github/",
- "dockerfile",
- ".yml",
- ".yaml",
- ".toml",
- ".ini",
- ".cfg",
- "lockfile",
- "package-lock",
- "uv.lock",
- "poetry.lock",
-)
-
-
-def _classify_hit_kind(target_path: str, page_type: str) -> str:
- """Bucket a hit into implementation / test / config / doc.
-
- Pages without a file behind them (decision records, repo overviews,
- onboarding pages — all have empty ``target_path``) are docs: letting
- them fall through to the path heuristics classified every decision
- record as "implementation", so ``kind="implementation"`` returned
- decision pages instead of filtering them out.
-
- ``test`` is the union of tests and test support here: someone asking for
- ``kind="implementation"`` does not want a ``conftest.py`` back, and someone
- asking for ``kind="test"`` does.
-
- Config wins over test, which the token list this replaced never had to
- decide: ``.github/workflows/tests.yml`` is a workflow whatever it is named,
- and dropping it from ``kind="config"`` would be the more surprising answer.
- """
- tp = (target_path or "").lower()
- if page_type in ("module_page", "symbol_spotlight") or tp.endswith(".md"):
- return "doc"
- if not tp or page_type not in FILE_ROW_TYPES:
- return "doc"
- if any(tok in tp for tok in _CONFIG_PATH_TOKENS):
- return "config"
- # Original case, not ``tp``: the camel (FooTest.java) and .NET project-dir
- # (Foo.Tests/) rules are deliberately case-sensitive.
- if is_test_related_path(target_path):
- return "test"
- return "implementation"
-
-
-# Every value :func:`_classify_hit_kind` can return; anything else can only ever
-# match nothing, which is why an unknown kind is dropped rather than filtered on.
-_VALID_KINDS = frozenset({"implementation", "test", "config", "doc"})
-
-
-def _filter_by_kind(output: list[dict], kind: str | None) -> list[dict]:
- """Keep only hits classified as ``kind`` (no-op when ``kind`` is falsy).
-
- Runs on the over-fetched list BEFORE the limit cut so the caller still
- gets up to ``limit`` results of the requested kind.
- """
- if not kind:
- return output
- return [
- item
- for item in output
- if _classify_hit_kind(item.get("target_path", ""), item.get("page_type", "")) == kind
- ]
-
-
-def _attach_paths(output: list[dict], page_info: dict) -> None:
- """Stamp canonical file paths and split symbol ids into their own field.
-
- A ``symbol_spotlight``'s target_path is ``file.py::Symbol``: a page id, and
- not a file path. Preserve it as ``symbol_id`` for ``get_symbol`` and expose
- only the file portion through ``target_path`` / ``file``.
-
- This lives in one function because it did not used to. The concept branch
- of ``search_codebase`` set ``file``; ``_search_single_repo``, which is what
- the hybrid and federated branches call, did not. A query carrying a
- CamelCase token routes to hybrid, so the tool's headline path kept serving
- page ids in a field read as a path. Measured on the dev-fix1 predictions:
- 45.4% of the python served paths still carried ``::``, and resolving them
- takes gold containment from 19 to 39 of 50 instances.
- """
- for item in output:
- item["target_path"] = page_info.get(item["page_id"], item.get("target_path", ""))
- if "::" in item["target_path"]:
- item["symbol_id"] = symbol_identity(item["target_path"])
- item["target_path"] = path_identity(item["target_path"].split("::", 1)[0])
- item["file"] = item["target_path"]
-
-
-def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:
- """Strip ``page_id`` from every result that can rebuild it, in place.
-
- A page id is ``compute_page_id(page_type, target_path)`` — literally
- ``f"{page_type}:{target_path}"`` (``core/generation/models.py``) — and both
- halves ship in the same row. So the field is 229-369 characters per
- response, 7.8-10.3% of the payload, restating two of its own siblings. It
- is ranking plumbing: every internal use above keys the fused dict on it,
- and none of that needs to reach the wire.
-
- Conditional rather than unconditional, and the condition is the derivation
- itself. ``_attach_paths`` writes ``target_path: ""`` for a hit whose Page
- row did not load, and a row like that cannot be rebuilt — so it keeps its
- id instead of losing one. The rule is "drop it only where it is provably
- redundant", which is also the check: if this ever stops firing, the
- invariant it rests on has changed.
-
- Consumers rebuild with the same expression; ``ui/src/chat/source-citations``
- does exactly that, and used to skip any row whose ``page_id`` was missing.
-
- ``add_row_paths`` runs first and deletes ``target_path`` wherever it equals
- the new ``path``, so the id's second half is ``target_path`` when present
- and ``path`` otherwise. Without that fallback an ordinary file row rebuilt
- as ``"file_page:"`` and kept its id.
- """
- for item in results:
- # A symbol page's id ends in ``::Symbol`` and ``_attach_paths`` moved that
- # half into ``symbol_id``, so ``target_path`` is only its file.
- target = (
- item.get("symbol_id")
- if item.get("page_type") == "symbol_spotlight"
- else None
- ) or item.get("target_path") or item.get("path", "")
- derived = f"{item.get('page_type', '')}:{target}"
- # A row for a file with no page has an id that names no page.
- if item.get("page_id") == derived or item.get("page_type") == PAGELESS_FILE:
- item.pop("page_id", None)
- return results
-
-
-def _page_language(ctx) -> str:
- """The language *ctx*'s structural page titles were generated in."""
- try:
- from repowise.core.repo_config import load_repo_config
-
- return str(load_repo_config(ctx.path).get("language") or "en")
- except Exception:
- return "en"
-
-
-def _slim_served_rows(
- rows: list[dict], languages: dict[str | None, str] | None = None
-) -> list[dict]:
- """The last step before a search reply leaves: one location per row.
-
- ``path`` (plus ``symbol_id`` on symbol rows) is the row's only location, so
- ``file``, a ``page_id`` that rebuilds from them, and a ``title`` that is just
- the structural label wrapped around them are all dropped. Each is dropped
- only where the caller can provably rebuild it. ``sources: ["fts"]`` goes when
- the reply's own ``_meta.semantic_search`` already says retrieval is
- full-text-only; an index with embeddings is unchanged.
-
- *languages* maps a row's ``repo`` alias (``None`` for a single-repo reply) to
- the language its titles were generated in.
- """
- add_row_paths(rows)
- _drop_derivable_page_ids(rows)
- fts_only = semantic_search_state() is False
- for row in rows:
- target = row.get("symbol_id") or row.get("path")
- if target and is_structural_title(
- (languages or {}).get(row.get("repo"), "en"),
- row.get("page_type", ""),
- target,
- row.get("title"),
- ):
- row.pop("title", None)
- if fts_only and row.get("sources") == ["fts"]:
- row.pop("sources", None)
- return rows
-
-
-def _serve_snippets(results: list[dict]) -> None:
- """Drop each snippet's leading page heading, after ranking has read it."""
- for item in results:
- if item.get("snippet"):
- item["snippet"] = strip_leading_headings(item["snippet"])
-
-
-def _drop_internal_ranking_fields(results: list[dict]) -> None:
- """Keep calibration diagnostics internal to the ranking pipeline."""
- for item in results:
- item.pop("_coverage", None)
- item.pop("_coverage_multiplier", None)
- item.pop("_confidence_score_factor", None)
- item.pop("_raw_score", None)
-
-
-async def _load_page_info(
- session, output: list[dict], *, with_git: bool = False
-) -> tuple[dict, set, dict]:
- """Batch-load target paths, tombstones, and git.
-
- Returns ``(page_info, tombstoned, git_map)`` where ``page_info`` maps
- page_id -> target_path, ``tombstoned`` is the set of tombstoned page_ids,
- and ``git_map`` maps file_path -> GitMetadata (empty unless ``with_git``).
- """
- page_ids = [item["page_id"] for item in output]
- res = await session.execute(
- select(Page.id, Page.target_path, Page.freshness_status).where(Page.id.in_(page_ids))
- )
- rows = res.all()
- page_info = {row[0]: row[1] for row in rows}
- tombstoned = {row[0] for row in rows if row[2] == "tombstone"}
-
- git_map: dict[str, GitMetadata] = {}
- if with_git:
- target_paths = [tp for tp in page_info.values() if tp]
- if target_paths:
- git_res = await session.execute(
- select(GitMetadata).where(GitMetadata.file_path.in_(target_paths))
- )
- git_map = {g.file_path: g for g in git_res.scalars().all()}
- return page_info, tombstoned, git_map
-
-
-def _apply_freshness_boost(item: dict, gm: GitMetadata | None) -> None:
- """Nudge a recently-active file up as a bounded tie-breaker (in place).
-
- Additive, not multiplicative: the fused relevance score is RRF-based, which
- compresses adjacent ranks to roughly ``_FRESHNESS_TIEBREAK`` apart. A
- percentage boost on that scale would let a fresh rank-2 hit leapfrog a
- stale rank-0 hit, overriding retrieval relevance with recency. Sized below
- one rank step, freshness only orders hits retrieval already ranks close
- together.
- """
- if not gm or not item.get("relevance_score"):
- return
- c30 = gm.commit_count_30d or 0
- c90 = gm.commit_count_90d or 0
- if c30 > 0:
- recency = 1.0
- elif c90 > 0:
- recency = 0.5
- else:
- recency = 0.0
- item["relevance_score"] = round(item["relevance_score"] + _FRESHNESS_TIEBREAK * recency, 4)
-
-
-def _assign_confidence(output: list[dict], score_key: str, target_key: str) -> None:
- """Derive ``target_key`` for each item from its ``score_key`` position (in place)."""
- if not output:
- return
- max_score = max((item.get(score_key) or 0) for item in output)
- for item in output:
- raw = item.get(score_key) or 0
- item[target_key] = round(raw / max_score, 2) if max_score > 0 else 0.0
-
-
-async def _wait_for_vector_store(ctx) -> None:
- """Block (bounded) until the vector store signals readiness, if it tracks it."""
- if ctx.vector_store_ready is not None:
- with contextlib.suppress(TimeoutError):
- await asyncio.wait_for(ctx.vector_store_ready.wait(), timeout=30.0)
-
-
-async def _safe_fts(ctx, query: str, limit: int) -> list:
- """FTS search, bounded and failure-swallowing (returns [] on error/timeout)."""
- if ctx.fts is None:
- return []
- with contextlib.suppress(Exception):
- return await asyncio.wait_for(ctx.fts.search(query, limit=limit), timeout=5.0)
- return []
-
-
-async def _safe_vector(ctx, query: str, limit: int) -> list:
- """Vector search, bounded and failure-swallowing (returns [] on error/timeout).
-
- Readiness is awaited by the caller (``_wait_for_vector_store``) before the
- fused retrieve runs, so this path does not re-wait.
-
- Returns nothing on a keyless index. ``store_has_semantic_vectors`` explains
- why the mock's vectors cannot be ranked on; the reason this is enforced here
- rather than in ``_fused_retrieve`` is that this function is the only place
- the vector leg is entered, so a later caller inherits the guard.
- """
- if ctx.vector_store is None or not store_has_semantic_vectors(ctx.vector_store):
- return []
- try:
- return await asyncio.wait_for(
- ctx.vector_store.search(query, limit=limit), timeout=vector_search_timeout_s()
- )
- except TimeoutError:
- # Not suppressed silently: a timeout here drops the whole semantic leg
- # and the caller returns full-text-only hits that look like a normal
- # result set, which is how #1678 read as "semantic search finds nothing".
- _log.warning(
- "Vector search exceeded its %gs budget; returning no semantic hits. "
- "Raise it with %s=<seconds>.",
- vector_search_timeout_s(),
- _VECTOR_TIMEOUT_ENV,
- )
- except Exception:
- _log.debug("Vector search failed; returning no semantic hits", exc_info=True)
- return []
-
-
-def _symbol_leg_target(fused: dict[str, dict], page_id: str) -> str:
- """The fused page a symbol-leg file page credits: the best page of its file.
-
- The leg names files, while the page legs may have reached the same file
- through a symbol page. Crediting only the file page id would leave that
- hit unlifted and add a weak twin that collapsing then drops.
- """
- path = page_id.partition(":")[2]
- best = None
- for pid, entry in fused.items():
- if file_path_of(entry["page_type"], pid.partition(":")[2]) != path:
- continue
- if best is None or entry["_rrf"] > fused[best]["_rrf"]:
- best = pid
- return best or page_id
-
-
-def _fused_entry(r) -> dict:
- """Seed a fused-result dict from a retriever hit (RRF score added by caller)."""
- entry = {
- "page_id": r.page_id,
- "title": r.title,
- "page_type": r.page_type,
- "snippet": r.snippet,
- "_rrf": 0.0,
- "_sources": set(),
- }
- if r.page_type == PAGELESS_FILE:
- # No page row to load it from later.
- entry["target_path"] = pageless_path(r.page_id)
- return entry
-
-
-async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | None) -> list[dict]:
- """Retrieve by fusing FTS, vector and symbol search via Reciprocal Rank Fusion.
-
- The retrievers run in parallel; a hit's score is the sum of
- ``1/(rank + k)`` over the retrievers that surfaced it, scaled into the
- BM25 range the downstream relevance gates were tuned against. Each hit
- carries a ``sources`` list (``"fts"``, ``"vector"``, ``"symbol"``,
- ``"filename"``): a page found by several retrievers is a stronger match
- than one found by one mode. The symbol and filename legs also name files
- that have no page, as ``page_type: "file"`` rows.
-
- This mirrors get_answer's ``hybrid_retrieve`` so the two entry points rank
- the same corpus identically. The prior path ran vector search and fell
- back to FTS only when vector returned nothing, so a page FTS ranked highly
- but vector missed never surfaced here.
-
- A hit is admitted only when its raw retriever score clears
- ``_MIN_RELEVANCE_SCORE`` — the fused score is rank-based and would
- otherwise sit well above the floor for every window hit, letting an
- off-topic query return its nearest-but-unrelated neighbours. Rank position
- (for RRF) is the hit's place in the retriever's own list, unchanged by the
- floor.
- """
- fts_results, vec_results, sym_results, name_results = await asyncio.gather(
- _safe_fts(ctx, query, fetch_limit),
- _safe_vector(ctx, query, fetch_limit),
- _safe_symbol_search(ctx, query, pageless=True),
- _safe_filename_search(ctx, query),
- )
-
- fused: dict[str, dict] = {}
- for rank, r in enumerate(vec_results):
- if r.score < _MIN_RELEVANCE_SCORE:
- continue
- entry = fused.setdefault(r.page_id, _fused_entry(r))
- entry["_rrf"] += 1.0 / (rank + _RRF_K)
- entry["_sources"].add("vector")
- for rank, r in enumerate(fts_results):
- if r.score < _MIN_RELEVANCE_SCORE:
- continue
- entry = fused.setdefault(r.page_id, _fused_entry(r))
- entry["_rrf"] += 1.0 / (rank + _RRF_K)
- entry["_sources"].add("fts")
- # Files whose symbol names the query's words spell, paged or not. Weighted
- # below the page legs, as in get_answer; it has no raw score to floor.
- for rank, r in leg_ranks(sym_results):
- entry = fused.setdefault(_symbol_leg_target(fused, r.page_id), _fused_entry(r))
- entry["_rrf"] += 1.0 / (rank + _SYMBOL_LEG_RRF_K)
- entry["_sources"].add("symbol")
- # Files whose file name the query's words spell, weighted as a name match.
- # It adds files the other legs missed and never reorders a page they found.
- # A name of three or more words the query spells in full weighs like a page hit.
- for rank, (r, full) in leg_ranks(name_results):
- target = _symbol_leg_target(fused, r.page_id)
- if target in fused and r.page_type != PAGELESS_FILE:
- continue
- entry = fused.setdefault(target, _fused_entry(r))
- entry["_rrf"] += 1.0 / (rank + (_RRF_K if full else _FILENAME_LEG_RRF_K))
- entry["_sources"].add("filename")
-
- output: list[dict] = []
- for entry in fused.values():
- if page_type and entry["page_type"] != page_type:
- continue
- entry["relevance_score"] = round(entry.pop("_rrf") * _RRF_SCORE_SCALE, 4)
- entry["sources"] = sorted(entry.pop("_sources"))
- output.append(entry)
- output.sort(key=lambda item: item["relevance_score"], reverse=True)
- return output
-
-
-async def _search_single_repo(
- ctx, query: str, limit: int, page_type: str | None, kind: str | None = None
-) -> list[dict]:
- """Run search against a single repo context.
-
- Fuses FTS and vector retrieval via RRF (see ``_fused_retrieve``), so each
- hit carries a ``sources`` list naming the retrievers that surfaced it
- rather than a single backend label.
-
- The ``kind`` filter runs here, on the over-fetched list and BEFORE the
- limit cut — filtering after per-repo truncation returned fewer than
- ``limit`` results (frequently zero) in the federated path.
- """
- await _wait_for_vector_store(ctx)
-
- fetch_limit = _fetch_limit_for(limit, kind)
- output = await _fused_retrieve(ctx, query, fetch_limit, page_type)
-
- _downweight_decisions(output, query)
- output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)
-
- # Attach target_path and drop excluded hits per repo (so federated search
- # honours each repo's own exclude_patterns) before ranking. Runs on the
- # over-fetched list so the kind filter below has real headroom. Load
- # precedes the sort/demotion so they see the page metadata they key on
- # (test demotion needs target_path to classify a test file page).
- if output:
- async with get_session(ctx.session_factory) as session:
- page_info, tombstoned, _ = await _load_page_info(session, output)
- path_index = await file_path_index(session, (await _get_repo(session)).id)
- output = [item for item in output if item["page_id"] not in tombstoned]
- _attach_paths(output, page_info)
- output = filter_dicts_by_key(output, "target_path", _get_exclude_spec(ctx.path))
- boost_named_paths(
- output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"
- )
-
- _downweight_test_pages(output, query)
- _sort_demoting_noise(output, query)
- output = _dedup_decisions(output)
-
- output = _filter_by_kind(output, kind)
- return _collapse_by_file(output)[:limit]
-
-
-def _neg_score(key: str):
- """Best-first sort key on a row's *key* score, for :func:`merge_ranked`."""
- return lambda item: -(item.get(key) or 0.0)
-
-
-async def _federated_search(
- query: str, limit: int, page_type: str | None, kind: str | None, fed: Federation
-) -> dict:
- """Search across all repos using Reciprocal Rank Fusion."""
-
- async def _one_repo(ctx) -> list[dict]:
- repo_results = await _search_single_repo(ctx, query, limit, page_type, kind)
- for rank, item in enumerate(repo_results):
- item["repo"] = ctx.alias
- item["rrf_score"] = 1.0 / (rank + 60) # RRF constant k=60
- return repo_results
-
- answered = await fed.run(_one_repo)
- contexts = [ctx for ctx, _ in answered]
- all_results = merge_ranked(
- (rows for _, rows in answered), _neg_score("rrf_score"), per_repo_cap=limit
- )
- output = all_results[:limit]
-
- # Derive confidence from RRF position
- _assign_confidence(output, "rrf_score", "confidence_score")
-
- # No single indexed commit to compare a live HEAD against: per-repo
- # freshness rides on ``repos_stale`` and the rows' ``stale`` flag instead.
- response: dict = {"results": output, "_meta": _build_meta()}
- fed.annotate(response, output)
- # Drawn from every repo's ranked list, not the merged cut: a workspace
- # search that spends its window on module pages should still be able to
- # name files.
- if candidates := file_candidates(all_results, limit=limit):
- response["candidates"] = candidates
- # Last, so nothing above has to know the field is on its way out.
- _slim_served_rows(output, {ctx.alias: _page_language(ctx) for ctx in contexts})
- _serve_snippets(output)
- return response
-
-
-def _refresh_served_targets(response: dict, _collector: OmissionCollector | None) -> None:
- """Re-derive target-scoped freshness from the hits that survived shedding.
-
- ``_meta.targets`` claims which files the response served. Shedding decides
- that, so this has to run after it or the claim names rows the caller never
- received.
- """
- if response.get("truncated") and isinstance(response.get("_meta"), dict):
- response["_meta"]["targets"] = _result_paths(response.get("results") or [])
-
-
-def _refresh_served_targets_final(response: dict) -> None:
- """Same claim, re-derived after the final size guard may have cut again."""
- _refresh_served_targets(response, None)
-
-
-register_post_shed("search_codebase", _refresh_served_targets)
-register_post_enforce("search_codebase", _refresh_served_targets_final)
-
-
-def _result_paths(results: list[dict]) -> list[str]:
- """File paths a result set serves, for target-scoped freshness.
-
- Symbol hits carry ``file``; page hits are resolved through their page type,
- so a module page's group key and an onboarding slot resolve to nothing
- rather than being handed on as if they were files. Freshness is per-file
- git metadata, and a directory has none, so a page id reaching here could
- only ever fail to match.
-
- An empty result set returns ``[]`` (meaning "no file content served",
- which suppresses the repo-level stale warning by design).
- """
- paths: list[str] = []
- for item in results:
- p = hit_file_path(item)
- if p:
- paths.append(p)
- return paths
-
-
-def _grep_hint_for(query: str, names: Container[str] | None = None) -> str | None:
- """Zero-result recovery hint for identifier-shaped queries, else ``None``.
-
- Only attached when the search produced nothing (see call sites) — a
- populated result set needs no escape hatch. Points back into the tool
- surface first; Grep is named only for the one job it genuinely wins:
- an exhaustive literal sweep of every usage (e.g. before a rename).
- """
- if _looks_like_exact_token(query):
- return (
- f"No indexed match for identifier {query!r}. Retry with "
- 'mode="symbol" (or check spelling/casing). ' + EXHAUSTIVE_SWEEP_HINT
- )
- if idents := _embedded_identifiers(query, names):
- shown = ", ".join(repr(t) for t in idents[:3])
- return (
- f"Query names identifier(s) {shown} but nothing matched. Search "
- 'the identifier alone with mode="symbol", then pipe the hit '
- "into get_symbol for its body. " + EXHAUSTIVE_SWEEP_HINT
- )
- return None
-
-
-async def _per_repo(repo: str | None, fed: Federation | None, fn) -> list[tuple[Any, Any]]:
- """``(context, fn(context))`` for the one repo asked for, or every repo via *fed*."""
- if fed is not None:
- return await fed.run(fn)
- ctx = await _resolve_repo_context(repo)
- return [(ctx, await fn(ctx))]
-
-
-def _lead_candidates(response: dict, issue: IssueFiles | None, limit: int) -> None:
- """Trace files lead ``candidates``. A named identifier's defining file only
- fills a free slot after the ranked ones: ranked above them, it displaces
- better hits on ordinary questions. Capped at ``limit``."""
- if issue is None or not (issue.traced or issue.named):
- return
- ranked = [c["path"] for c in response.get("candidates") or []]
- paths = list(dict.fromkeys(issue.traced + ranked + issue.named))
- response["candidates"] = [{"path": p} for p in paths[:limit]]
-
-
-def _tag_repo(items: list[dict], ctx, multi: bool) -> None:
- if multi:
- for item in items:
- item["repo"] = ctx.alias
-
-
-def _missing_named_symbols(
- candidates: list[str], symbols: list[dict], canonical: bool, concepts: list[dict]
-) -> list[str]:
- """Code-shaped names the query asks after that no returned symbol matches
- exactly (or returned module path carries): they do not exist here. Judged
- per name, so an indexed name beside a missing one hides nothing. Their
- fuzzy neighbours would stand in for them, so the caller keeps only exact
- symbols; the pages are marked here as related to the question, not the
- symbol."""
- if canonical:
- return []
- page_paths = [c.get("target_path") or "" for c in concepts]
- missing = [
- c
- for c in candidates
- if _looks_like_code_name(c)
- and not _has_exact_symbol([c], symbols)
- and not _names_a_path(c, page_paths)
- ]
- if missing:
- _mark_not_the_named_symbol(concepts)
- return missing
-
-
-@dataclass(frozen=True)
-class _SearchPlan:
- """One structured search: the caller's arguments plus the derived query."""
-
- query: str
- limit: int
- page_type: str | None
- kind: str | None
- symbol_kind: str | None
- mode: str
- names: Container[str] | None
- canonical_symbol: tuple[Any, str] | None
- symbol_query: str
- fetch: int
- multi: bool
-
-
-@dataclass
-class _Pools:
- """Each row kind merged and ranked across the repos searched."""
-
- contexts: list[Any]
- symbols: list[dict]
- spare: list[dict]
- files: list[dict]
- indexed_paths: list[str]
- concepts: list[dict]
-
-
-def _symbol_query_for(
- query: str, mode: str, names: Container[str] | None, canonical_symbol: Any
-) -> str:
- """The text the symbol scorer ranks on."""
- # Scored on the identifiers, not the prose: words like "is" would match
- # ``is_ci`` and bury the symbol the question names.
- symbol_query = query
- if canonical_symbol:
- symbol_query = canonical_symbol[1]
- if mode == "hybrid":
- _idents = _embedded_identifiers(query, names)
- # No identifier: content words, else the raw text.
- symbol_query = (
- " ".join(_idents) if _idents else " ".join(content_terms(query)) or query
- )
- return symbol_query
-
-
-def _symbol_fetch(mode: str, limit: int) -> int:
- """How many symbol rows each repo fetches."""
- # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every
- # candidate so ``fuzzy_omitted`` can count what an exact hit hides.
- fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit
- if mode == "symbol":
- fetch = _MAX_CANDIDATES
- return fetch
-
-
-async def _search_one_repo(
- ctx: Any, plan: _SearchPlan
-) -> tuple[list[dict], list[dict], list[str], list[dict]]:
- """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""
- s: list[dict] = []
- f: list[dict] = []
- paths: list[str] = []
- c: list[dict] = []
- if plan.mode in ("symbol", "hybrid"):
- s = await search_symbols_single(
- ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind
- )
- if plan.mode == "path":
- f = await search_paths_single(ctx, plan.query, plan.limit)
- async with get_session(ctx.session_factory) as session:
- paths = (await file_path_index(session, (await _get_repo(session)).id)).paths
- if plan.mode == "hybrid":
- c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)
- for item in c:
- item["type"] = "page"
- await attach_hit_symbols(ctx, plan.query, c)
- for rows in (s, f, c):
- _tag_repo(rows, ctx, plan.multi)
- return s, f, paths, c
-
-
-def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:
- """Merge each repo's rows into one ranked pool per row kind."""
- parts = [part for _, part in answered]
- return _Pools(
- contexts=[ctx for ctx, _ in answered],
- symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),
- # Rows past each repo's limit only backfill slots that collapsing frees.
- spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),
- files=merge_ranked((p[1] for p in parts), _neg_score("score")),
- indexed_paths=[path for p in parts for path in p[2]],
- concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),
- )
-
-
-def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:
- """Keep exact name and path matches ahead of their fuzzy neighbours."""
- if plan.mode == "symbol":
- pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)
- elif plan.mode == "hybrid" and candidates:
- pools.symbols = _protect_named_symbols(candidates, pools.symbols)
- if plan.mode == "path":
- pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)
-
-
-def _symbol_mode_results(
- plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
-) -> tuple[list[dict], int]:
- """Symbol mode rows, and how many fuzzy names an exact hit hides."""
- # An exact hit is the answer; its fuzzy neighbours are only counted.
- fuzzy_omitted = 0
- pool = pools.symbols + pools.spare
- named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []
- if named:
- canonical_symbol = plan.canonical_symbol
- ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()
- shown = {id(s) for s in named}
- fuzzy_omitted = sum(
- 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()
- )
- return (named or pools.symbols)[: plan.limit], fuzzy_omitted
-
-
-def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:
- """Hybrid rows: symbol matches interleaved with concept pages for new files."""
- window = _hybrid_window(
- plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names
- )
- # One row per file; freed slots fill with the next pages, then symbols.
- served = {id(item) for item in window}
- rest = [
- item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served
- ]
- results = _collapse_by_file(window + rest)[: plan.limit]
- # Non-file pages leave the window unrefilled unless the caller asked for
- # pages by type or kind="doc".
- if not plan.page_type and plan.kind != "doc":
- results = [item for item in results if hit_file_path(item)]
- return results
-
-
-def _select_results(
- plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
-) -> tuple[list[dict], int]:
- """The served rows for the plan's mode, and the symbol-mode fuzzy count."""
- if plan.mode == "symbol":
- return _symbol_mode_results(plan, pools, candidates, exact)
- if plan.mode == "path":
- return pools.files[: plan.limit], 0
- return _hybrid_mode_results(plan, pools, exact), 0
-
-
-async def _single_repository(contexts: list[Any], multi: bool) -> Any:
- """The repository row a single-repo search ran against, else None."""
- if multi or not contexts:
- return None
- async with get_session(contexts[0].session_factory) as session:
- return await _get_repo(session)
-
-
-def _exact_match_note(
- candidates: list[str],
- exact: bool,
- missing: list[str],
- fuzzy_omitted: int,
- has_results: bool,
- grep_hint: str | None,
-) -> str | None:
- """The note qualifying an identifier query's results, if one is owed."""
- if missing:
- shown = ", ".join(repr(c) for c in missing[:3])
- # An empty window's grep_hint already carries the sweep advice.
- pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""
- sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT
- return (
- f"No indexed symbol is named {shown}, so no symbol is returned "
- f"for it.{pages} Recheck the spelling, or search a shorter part "
- f"of the name.{sweep}"
- )
- if not exact:
- shown = ", ".join(repr(c) for c in candidates[:3])
- return (
- f"No indexed symbol exactly matches {shown}. The results are "
- "fuzzy neighbours ranked by token overlap — confirm a hit names "
- "what you meant before relying on it. If you expected an exact "
- "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT
- )
- if fuzzy_omitted:
- return (
- f"{fuzzy_omitted} other symbols contain this name; search a "
- "longer or partial name to list them."
- )
- return None
-
-
-def _match_named(
- plan: _SearchPlan, pools: _Pools, candidates: list[str]
-) -> tuple[bool, list[str]]:
- """Whether a returned symbol names the query exactly, and the names none does.
-
- Missing names leave only exact symbols in *pools*.
- """
- exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False
- missing = _missing_named_symbols(
- candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts
- )
- if missing:
- pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]
- pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]
- return exact, missing
-
-
-def _attach_match_notes(
- response: dict,
- candidates: list[str],
- exact: bool,
- missing: list[str],
- grep_hint: str | None,
-) -> None:
- """Exact-match flag, its note, and the grep fallback for an empty window."""
- results = response["results"]
- # Fuzzy neighbours of a missing name look authoritative, so say so.
- if candidates:
- response["exact_match"] = exact and not missing
- note = _exact_match_note(
- candidates,
- exact,
- missing,
- response.get("fuzzy_omitted", 0),
- bool(results),
- grep_hint,
- )
- if note:
- response["note"] = note
- if grep_hint and not results:
- response["grep_hint"] = grep_hint
-
-
-async def _structured_search(
- query: str,
- limit: int,
- page_type: str | None,
- kind: str | None,
- symbol_kind: str | None,
- repo: str | None,
- mode: str,
- grep_hint: str | None,
- names: Container[str] | None = None,
- fed: Federation | None = None,
-) -> dict:
- """Run symbol / path / hybrid search, across repos when federated, and shape the response.
-
- ``grep_hint`` is attached only when nothing was found.
- """
- multi = fed is not None and fed.multi
- canonical_symbol = _canonical_symbol_query(query)
- plan = _SearchPlan(
- query=query,
- limit=limit,
- page_type=page_type,
- kind=kind,
- symbol_kind=symbol_kind,
- mode=mode,
- names=names,
- canonical_symbol=canonical_symbol,
- symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),
- fetch=_symbol_fetch(mode, limit),
- multi=multi,
- )
-
- async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:
- return await _search_one_repo(ctx, plan)
-
- pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)
- candidates = _identifier_candidates(query, mode, names)
- _protect_exact(plan, pools, candidates)
- exact, missing = _match_named(plan, pools, candidates)
-
- results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)
- repository = await _single_repository(pools.contexts, multi)
-
- response: dict = {
- "results": results,
- "mode": mode,
- "_meta": _build_meta(repository=repository, targets=_result_paths(results)),
- }
- if fuzzy_omitted:
- response["fuzzy_omitted"] = fuzzy_omitted
- # Not ``candidates``: that name holds the query's identifiers the note quotes.
- if file_cands := file_candidates(results, limit=limit):
- response["candidates"] = file_cands
- _attach_match_notes(response, candidates, exact, missing, grep_hint)
- if fed is not None:
- fed.annotate(response, results)
- # Last: slims fields the steps above still read.
- _slim_served_rows(
- results,
- {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},
- )
- _serve_snippets(results)
- return response
-
-
- surface_order=40,
- artifact_type="search_results",
- presentation="search_results",
- evidence_basis="inferred",
- recipes=(
- ToolRecipe(
- "find_raw_hits",
- 'search_codebase(query="identifier or path", mode="auto")',
- ("search_codebase",),
- ),
- ),
-)
-async def search_codebase(
- query: str = "",
- limit: int = 5,
- page_type: str | None = None,
- kind: str | None = None,
- repo: str | None = None,
- mode: str = "auto",
- symbol_kind: str | None = None,
- pattern: str | None = None,
-) -> dict:
- """Find code by concept, symbol, or path — hybrid codebase search.
-
- For questions (how/where/why), call get_answer instead: it runs this
- retrieval internally and returns a cited answer. Use this for raw
- ranked hits: enumerating matches, resolving an identifier to a symbol_id,
- or scoping get_context.
-
- mode="auto" routes by query shape: identifiers search symbols (symbol_id,
- path, line bounds for get_symbol), paths resolve files (for get_context),
- prose runs wiki-semantic search, and mixed queries run hybrid (symbol hits,
- then file-backed pages only). Decision records rank below file pages
- unless the query is why-shaped.
-
- Rows naming a file carry `path`; concept pages add `symbols`
- (name:line) the query matches; the top 3 add `matched_lines`
- (line, text), sample lines sharing its words. `candidates` lists up to `limit`
- distinct files to Read, best first. Identifier or literal queries also
- return `lines` (path, line, kind, text; definitions first) read from live
- files; when `complete` is true they are every match, so no grep is needed.
-
- Args:
- query: identifier, path, or natural language.
- limit: max results (default 5); distinct files outside mode="symbol"
- (same-file symbols in `symbols`).
- page_type: file_page (per-file docs) or module_page (subsystem
- pages); any stored page type filters.
- kind: implementation | test | config | doc (concept/symbol modes).
- repo: alias, or "all" for workspace-wide.
- mode: auto | concept | symbol | path | hybrid.
- symbol_kind: filter symbol hits (function|class|method|...).
- pattern: alias for query.
- """
- ignored: list[dict[str, Any]] = []
- # Hosts that defer tool schemas let a model guess grep's argument name.
- if pattern is not None and not query.strip():
- query = pattern
- elif pattern is not None:
- ignored.append(
- {"argument": "pattern", "values": [pattern], "valid": [], "superseded_by": "query"}
- )
- if not query.strip():
- return {
- "results": [],
- "error": "search_codebase requires `query` (or its alias `pattern`).",
- "_meta": _build_meta(),
- }
- # An unknown kind used to take the same ``return False`` as a kind that is
- # simply inapplicable, so a typo and a real empty result looked identical.
- kind = resolve_enum_argument(kind, _VALID_KINDS, argument="kind", ignored=ignored)
- # An unknown mode used to become ``auto`` with nothing said, so the caller
- # got a different search than it asked for. Modes have always been
- # case-insensitive; only a genuine miss is dropped, and under its own spelling.
- if mode is not None and mode.lower() in _VALID_MODES:
- mode = mode.lower()
- mode = resolve_enum_argument(mode, _VALID_MODES, argument="mode", ignored=ignored)
- # One fan-out state for the whole call: a repo that fails one stage is
- # skipped by the next and reported once.
- fed = Federation() if repo == "all" else None
- # Loaded once so routing, candidates, ordering and the hint validate alike.
- names: set[str] = set()
- for _, repo_names in await _per_repo(repo, fed, lambda ctx: indexed_names([ctx], query)):
- names |= repo_names
- grep_hint = _grep_hint_for(query, names)
- resolved_mode = _resolve_mode(query, mode, names)
- # Single repo only: a federated list has no one tree to name files in.
- issue: IssueFiles | None = None
- if repo != "all":
- try:
- issue = await issue_files(await _resolve_repo_context(repo), query, names)
- except Exception:
- _log.warning("search_codebase: issue file lookup failed", exc_info=True)
-
- if resolved_mode in ("symbol", "path", "hybrid"):
- structured = await _structured_search(
- query, limit, page_type, kind, symbol_kind, repo, resolved_mode, grep_hint, names, fed
- )
- _lead_candidates(structured, issue, limit)
- await attach_line_hits(structured, query, resolved_mode, names, repo)
- attach_ignored_arguments(structured, ignored)
- return structured
-
- if fed is not None:
- # kind is filtered per-repo inside _search_single_repo, before each
- # repo's limit cut, so the fused list is already kind-pure and full.
- federated = await _federated_search(query, limit, page_type, kind, fed)
- if grep_hint and not federated.get("results"):
- federated["grep_hint"] = grep_hint
- attach_ignored_arguments(federated, ignored)
- return federated
-
- ctx = await _resolve_repo_context(repo)
-
- async with get_session(ctx.session_factory) as session:
- # Validate repo exists in DB
- repository = await _get_repo(session)
-
- await _wait_for_vector_store(ctx)
-
- # Fuse FTS + vector via RRF (see _fused_retrieve) so a page either mode
- # ranks highly surfaces here, matching get_answer's retrieval. Always
- # over-fetch (see _fetch_limit_for): post-filters trim hits, and decision
- # down-weighting needs file pages inside the window to promote.
- fetch_limit = _fetch_limit_for(limit, kind)
- output = await _fused_retrieve(ctx, query, fetch_limit, page_type)
-
- _downweight_decisions(output, query)
- output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)
-
- # Batch-lookup page target paths for the kind filter + git freshness boost
- if output:
- async with get_session(ctx.session_factory) as session:
- page_info, tombstoned, git_map = await _load_page_info(session, output, with_git=True)
- path_index = await file_path_index(session, repository.id)
- # Tombstoned pages document deleted/renamed files — never results.
- output = [item for item in output if item["page_id"] not in tombstoned]
-
- # Attach target_path to each item so the kind filter (path-prefix
- # heuristic) and downstream get_context callers can act on it, and
- # ``file`` wherever the page id is symbol-qualified.
- _attach_paths(output, page_info)
-
- output = filter_dicts_by_key(output, "target_path", _get_exclude_spec(ctx.path))
-
- for item in output:
- # Freshness boost: recently-active files rank higher
- target_path = page_info.get(item["page_id"])
- gm = git_map.get(target_path) if target_path else None
- _apply_freshness_boost(item, gm)
-
- # Re-sort by adjusted relevance with retrieval noise (decisions on
- # non-why queries, test pages on non-test queries) hard-demoted, then
- # collapse near-duplicate decisions to one.
- output = rerank_pages_first(output, query, score_key="relevance_score", floor=0.5)
- # After the coverage rerank, whose window-relative weights cannot tell
- # a word rare across the repo from one rare in these few hits.
- boost_named_paths(
- output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"
- )
- _downweight_test_pages(output, query)
- _sort_demoting_noise(output, query)
- output = _dedup_decisions(output)
-
- output = _filter_by_kind(output, kind)
- # A file page and a symbol page of one file are one place to look.
- output = _collapse_by_file(output)
- # Files the page retrievers structurally cannot see (a private helper, a
- # local name, anything a file page's public-symbol table omits) get the
- # weakest tail slots. No-op when the symbol leg names nothing new.
- output = _reserve_symbol_tail(output, limit, kind)
- _drop_internal_ranking_fields(output)
- # The full ranked pool, kept before the caller's cut so ``candidates``
- # below can reach past it. See its comment for why that matters.
- ranked = list(output)
- output = output[:limit]
- await attach_hit_symbols(ctx, query, output)
-
- # Derive confidence_score from relative position in the result set.
- _assign_confidence(output, "relevance_score", "confidence_score")
-
- response: dict = {
- "results": output,
- "_meta": _build_meta(repository=repository, targets=_result_paths(output)),
- }
- if candidates := file_candidates(ranked, limit=limit):
- response["candidates"] = candidates
- _lead_candidates(response, issue, limit)
- if grep_hint and not output:
- response["grep_hint"] = grep_hint
- await attach_line_hits(response, query, resolved_mode, names, repo)
- attach_ignored_arguments(response, ignored)
- # Last, so nothing above has to know the field is on its way out.
- _slim_served_rows(output, {None: _page_language(ctx)})
- _serve_snippets(output)
- return response
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/__init__.py b/packages/server/src/repowise/server/mcp_server/tool_search/__init__.py
new file mode 100644
index 000000000..bb3109a03
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/__init__.py
@@ -0,0 +1,186 @@
+"""MCP Tool 5: search_codebase, hybrid codebase search.
+
+``tool.py`` holds the registered tool; the sibling modules hold its stages.
+Importing the package registers the tool. Every name the single module had
+is re-exported here, and a stage calls a name tests patch through the package
+(``_pkg.<name>``), never by bare name, or the patch stops reaching it.
+"""
+
+# ruff: noqa: F401
+
+from __future__ import annotations
+
+from sqlalchemy import select
+
+from repowise.core.generation.structural_labels import is_structural_title
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.models import (
+ GitMetadata,
+ Page,
+)
+from repowise.core.persistence.search import strip_leading_headings
+from repowise.core.providers.embedding import store_has_semantic_vectors
+from repowise.core.registry import ToolRecipe
+from repowise.core.registry import mcp_tool_registry as mcp
+from repowise.core.test_paths import is_test_path, is_test_related_path
+from repowise.server.mcp_server._answer_pipeline import (
+ _FILENAME_LEG_RRF_K,
+ _RRF_K,
+ _RRF_SCORE_SCALE,
+ _SYMBOL_LEG_RRF_K,
+ _safe_filename_search,
+ _safe_symbol_search,
+ leg_ranks,
+)
+from repowise.server.mcp_server._budget import (
+ OmissionCollector,
+ register_post_enforce,
+ register_post_shed,
+)
+from repowise.server.mcp_server._federation import Federation, merge_ranked
+from repowise.server.mcp_server._helpers import (
+ _VECTOR_TIMEOUT_ENV,
+ _get_exclude_spec,
+ _get_repo,
+ _resolve_repo_context,
+ attach_ignored_arguments,
+ filter_dicts_by_key,
+ resolve_enum_argument,
+ vector_search_timeout_s,
+)
+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+from repowise.server.mcp_server._line_hits import attach_line_hits
+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT, semantic_search_state
+from repowise.server.mcp_server._meta import build_meta as _build_meta
+from repowise.server.mcp_server._page_paths import (
+ FILE_ROW_TYPES,
+ PAGELESS_FILE,
+ add_row_paths,
+ file_candidates,
+ file_path_of,
+ hit_file_path,
+ pageless_path,
+)
+from repowise.server.mcp_server._query_shape import (
+ _DECISION_DOWNWEIGHT,
+ _MIN_RELEVANCE_SCORE,
+ _VALID_MODES,
+ NOT_THE_NAMED_SYMBOL,
+ _canonical_symbol_query,
+ _embedded_identifiers,
+ _fetch_limit_for,
+ _has_exact_symbol,
+ _identifier_candidates,
+ _is_why_shaped,
+ _looks_like_code_name,
+ _looks_like_exact_token,
+ _mark_not_the_named_symbol,
+ _names_a_path,
+ _qual_norm,
+ _resolve_mode,
+ _symbol_matches_name,
+ path_tokens,
+)
+from repowise.server.mcp_server._query_terms import content_terms
+from repowise.server.mcp_server._references import path_identity, symbol_identity
+from repowise.server.mcp_server._retrieval_rank import (
+ boost_named_paths,
+ rerank_pages_first,
+)
+from repowise.server.mcp_server.tool_search.ordering import (
+ _FILE_WINDOW_OVERFETCH,
+ _MERGED_SYMBOL_CAP,
+ _collapse_by_file,
+ _hybrid_window,
+ _interleave_hybrid,
+ _prose_dominates,
+ _protect_exact_paths,
+ _protect_exact_symbols,
+ _protect_named_symbols,
+)
+from repowise.server.mcp_server.tool_search.ranking import (
+ _CONFIG_PATH_TOKENS,
+ _SYMBOL_TAIL_DIVISOR,
+ _TEST_DOWNWEIGHT,
+ _TEST_QUERY_RE,
+ _VALID_KINDS,
+ _classify_hit_kind,
+ _dedup_decisions,
+ _downweight_decisions,
+ _downweight_test_pages,
+ _filter_by_kind,
+ _is_test_page,
+ _is_test_query,
+ _non_decision_fallback,
+ _norm_decision_title,
+ _rescue_all_decision_window,
+ _reserve_symbol_tail,
+ _sort_demoting_noise,
+ _symbol_tail_reserve,
+)
+from repowise.server.mcp_server.tool_search.retrieval import (
+ _federated_search,
+ _fused_entry,
+ _fused_retrieve,
+ _neg_score,
+ _per_repo,
+ _safe_fts,
+ _safe_vector,
+ _search_single_repo,
+ _symbol_leg_target,
+ _wait_for_vector_store,
+)
+from repowise.server.mcp_server.tool_search.serve import (
+ _FRESHNESS_TIEBREAK,
+ _apply_freshness_boost,
+ _assign_confidence,
+ _attach_paths,
+ _drop_derivable_page_ids,
+ _drop_internal_ranking_fields,
+ _grep_hint_for,
+ _load_page_info,
+ _page_language,
+ _refresh_served_targets,
+ _refresh_served_targets_final,
+ _result_paths,
+ _serve_snippets,
+ _slim_served_rows,
+)
+from repowise.server.mcp_server.tool_search.structured import (
+ _attach_match_notes,
+ _exact_match_note,
+ _hybrid_mode_results,
+ _lead_candidates,
+ _match_named,
+ _merge_pools,
+ _missing_named_symbols,
+ _Pools,
+ _protect_exact,
+ _search_one_repo,
+ _SearchPlan,
+ _select_results,
+ _single_repository,
+ _structured_search,
+ _symbol_fetch,
+ _symbol_mode_results,
+ _symbol_query_for,
+ _tag_repo,
+)
+from repowise.server.mcp_server.tool_search.tool import (
+ _concept_search,
+ _indexed_names,
+ _issue_files,
+ _query_from_pattern,
+ _rank_concept_pages,
+ _resolve_kind_and_mode,
+ search_codebase,
+)
+from repowise.server.mcp_server.tool_search_symbols import (
+ _MAX_CANDIDATES,
+ IssueFiles,
+ file_path_index,
+ indexed_names,
+ issue_files,
+ search_paths_single,
+ search_symbols_single,
+)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/ordering.py b/packages/server/src/repowise/server/mcp_server/tool_search/ordering.py
new file mode 100644
index 000000000..a73aae833
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/ordering.py
@@ -0,0 +1,158 @@
+"""Exact-match protection and hybrid window ordering for search results."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Container, Sequence
+
+from repowise.server.mcp_server._page_paths import hit_file_path
+from repowise.server.mcp_server._query_shape import (
+ _canonical_symbol_query,
+ _embedded_identifiers,
+ _qual_norm,
+ _symbol_matches_name,
+ path_tokens,
+)
+
+
+def _protect_exact_symbols(query: str, symbols: list[dict]) -> list[dict]:
+ """Stable-partition an exact identifier or ``path::Symbol`` hit to the head."""
+ if not symbols:
+ return symbols
+ canonical = _canonical_symbol_query(query)
+ qnorm = query.strip().lower().replace("\\", "/")
+
+ def exact(item: dict) -> bool:
+ symbol_id = (item.get("symbol_id") or "").lower().replace("\\", "/")
+ if canonical:
+ return symbol_id == qnorm
+ return qnorm in {
+ (item.get("name") or "").lower(),
+ _qual_norm(item.get("qualified_name")),
+ }
+
+ protected = [item for item in symbols if exact(item)]
+ return protected + [item for item in symbols if not exact(item)]
+
+
+def _protect_named_symbols(candidates: list[str], symbols: list[dict]) -> list[dict]:
+ """Stable-partition symbols exactly named by identifiers embedded in prose."""
+ wanted = {candidate.strip().lower() for candidate in candidates if candidate.strip()}
+ wanted |= {_qual_norm(candidate) for candidate in candidates if candidate.strip()}
+
+ protected = [item for item in symbols if _symbol_matches_name(item, wanted)]
+ return protected + [item for item in symbols if not _symbol_matches_name(item, wanted)]
+
+
+def _protect_exact_paths(
+ query: str, files: list[dict], paths: Sequence[str] = ()
+) -> list[dict]:
+ """Stable-partition an exact path hit ahead of basename/fuzzy neighbours."""
+ tokens = path_tokens(query, paths) or [query]
+ wanted = {t.strip().lower().replace("\\", "/") for t in tokens}
+ protected = [item for item in files if (item.get("file") or "").lower() in wanted]
+ return protected + [item for item in files if (item.get("file") or "").lower() not in wanted]
+
+
+def _prose_dominates(query: str, identifiers: list[str]) -> bool:
+ """True when natural-language tokens outnumber the identifier tokens a query
+ carries: the query reads as prose that merely *mentions* a symbol, not a
+ symbol lookup dressed in a few words. Drives hybrid ordering below."""
+ ident_count = len(identifiers)
+ if ident_count == 0:
+ return False
+ total = len(re.findall(r"[A-Za-z0-9_]+", query))
+ return (total - ident_count) > ident_count
+
+
+def _interleave_hybrid(
+ query: str,
+ symbols: list[dict],
+ concepts: list[dict],
+ limit: int,
+ exact: bool,
+ names: Container[str] | None = None,
+) -> list[dict]:
+ """Order one hybrid result window from the two incomparable score scales.
+
+ Symbol SQL scores (~43, +100 on an exact-name hit) and concept relevance
+ (0 to 1) can't be merge-sorted, so we interleave by block. Default: symbols
+ lead, reserving up to half the window for concept pages so a flood of symbol
+ matches can't truncate every page out.
+
+ The exception is the score-scale trap. When NO returned symbol matches the
+ query's identifier exactly AND the query is mostly prose, leading with fuzzy
+ symbol hits buries the page the caller actually wants: the generic ``.get``
+ methods that outscore the ``answer.py`` page for "how does retrieval feed
+ synthesis in get_answer". There, concept pages lead and the fuzzy symbols
+ fall to the tail (nothing is dropped, only reordered within the window).
+ """
+ if not exact and concepts and _prose_dominates(query, _embedded_identifiers(query, names)):
+ reserved = min(len(symbols), limit // 2)
+ return (concepts[: max(1, limit - reserved)] + symbols)[:limit]
+ reserved = min(len(concepts), limit // 2)
+ return (symbols[: max(1, limit - reserved)] + concepts)[:limit]
+
+
+# Code-location windows over-fetch this many times ``limit`` so collapsing
+# same-file hits still leaves ``limit`` distinct files to serve.
+_FILE_WINDOW_OVERFETCH = 2
+# Other symbol names a collapsed row carries in ``symbols``.
+_MERGED_SYMBOL_CAP = 5
+
+
+def _collapse_by_file(hits: list[dict]) -> list[dict]:
+ """One row per (repo, file), best first; rows naming no file pass through.
+
+ The first row of a file wins. A later symbol hit in the same file folds
+ into the winner's ``symbols`` as ``name:line`` (capped) instead of taking a
+ slot of its own, so ``limit`` buys distinct files; past the cap a trailing
+ ``+N more`` entry counts the rest. Precedent:
+ ``symbol_backed_pages`` collapses the same way for the concept tail.
+ """
+ first_of: dict[tuple, dict] = {}
+ extra: dict[int, int] = {}
+ out: list[dict] = []
+ for hit in hits:
+ path = hit_file_path(hit)
+ if path is None:
+ out.append(hit)
+ continue
+ key = (hit.get("repo"), path)
+ first = first_of.get(key)
+ if first is None:
+ first_of[key] = hit
+ out.append(hit)
+ elif hit.get("type") == "symbol" and first.get("type") == "symbol":
+ merged = first.setdefault("symbols", [])
+ if len(merged) < _MERGED_SYMBOL_CAP:
+ merged.append(f"{hit.get('name')}:{hit.get('start_line')}")
+ else:
+ extra[id(first)] = extra.get(id(first), 0) + 1
+ for hit in out:
+ if id(hit) in extra:
+ hit["symbols"].append(f"+{extra[id(hit)]} more")
+ return out
+
+
+def _hybrid_window(
+ query: str,
+ symbols: list[dict],
+ concepts: list[dict],
+ limit: int,
+ exact: bool,
+ names: Container[str] | None = None,
+) -> list[dict]:
+ """The hybrid window, with no page for a file a shown symbol row covers.
+
+ Pages are dropped only for symbols the window actually shows, then the
+ window is rebuilt so the freed slots backfill. Concepts only shrink and the
+ symbol share only grows, so this settles in a few passes.
+ """
+ while True:
+ window = _interleave_hybrid(query, symbols, concepts, limit, exact, names)
+ shown = {item.get("file") for item in window if item.get("type") == "symbol"}
+ kept = [c for c in concepts if c.get("target_path") not in shown]
+ if len(kept) == len(concepts):
+ return window
+ concepts = kept
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/ranking.py b/packages/server/src/repowise/server/mcp_server/tool_search/ranking.py
new file mode 100644
index 000000000..2fd52cf49
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/ranking.py
@@ -0,0 +1,287 @@
+"""Noise demotion, decision rescue, symbol tail and kind filtering."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import re
+
+from repowise.core.test_paths import is_test_path, is_test_related_path
+
+# Patchable names are read off the package at call time so patches reach them.
+from repowise.server.mcp_server import tool_search as _pkg
+from repowise.server.mcp_server._helpers import vector_search_timeout_s
+from repowise.server.mcp_server._page_paths import FILE_ROW_TYPES
+from repowise.server.mcp_server._query_shape import (
+ _DECISION_DOWNWEIGHT,
+ _MIN_RELEVANCE_SCORE,
+ _is_why_shaped,
+)
+
+_log = logging.getLogger("repowise.mcp.search")
+
+
+def _downweight_decisions(output: list[dict], query: str) -> None:
+ """Scale decision_record relevance in place unless the query is why-shaped."""
+ if _is_why_shaped(query):
+ return
+ for item in output:
+ if item.get("page_type") == "decision_record" and item.get("relevance_score"):
+ item["relevance_score"] = round(item["relevance_score"] * _DECISION_DOWNWEIGHT, 4)
+
+
+# Test pages win retrieval on vocabulary shared with the code they exercise, but
+# are rarely a better first Read: demote them unless the query is about tests.
+_TEST_DOWNWEIGHT = 0.6
+
+_TEST_QUERY_RE = re.compile(
+ r"\b(test|tests|testing|tested|unit[\s-]?test|integration[\s-]?test|pytest|fixture|mock|spec)\b",
+ re.IGNORECASE,
+)
+
+
+def _is_test_query(query: str) -> bool:
+ """True when the query is explicitly about tests, so test pages rank naturally."""
+ return bool(_TEST_QUERY_RE.search(query))
+
+
+def _is_test_page(item: dict) -> bool:
+ """True when a hit is a whole-file row for a test file.
+
+ Tests only, deliberately not test support: a ``conftest.py`` or a fixture
+ module is often exactly what the person searching wanted, so it keeps its
+ rank. The ``kind`` filter counts both (see :func:`_classify_hit_kind`) —
+ asking for tests and asking to rank tests lower are different questions.
+ """
+ return item.get("page_type") in FILE_ROW_TYPES and is_test_path(item.get("target_path") or "")
+
+
+def _downweight_test_pages(output: list[dict], query: str) -> None:
+ """Scale test file_page relevance in place unless the query is about tests."""
+ if _is_test_query(query):
+ return
+ for item in output:
+ if item.get("relevance_score") and _is_test_page(item):
+ item["relevance_score"] = round(item["relevance_score"] * _TEST_DOWNWEIGHT, 4)
+
+
+def _sort_demoting_noise(output: list[dict], query: str) -> None:
+ """Sort by relevance, ranking retrieval noise below every real page.
+
+ Two classes crowd the top ranks on a plain implementation query and are
+ demoted absolutely for their query class (the relevance score still orders
+ each class among itself):
+
+ - decision records — short dense titles win cosine similarity by a margin
+ wider than the multiplicative down-weight. Rationale questions are
+ get_why's territory, so a why-shaped query ranks them naturally instead.
+ - test file pages — a test is rarely a better first Read than the code it
+ exercises. A query explicitly about tests ranks them naturally instead.
+ """
+ why = _is_why_shaped(query)
+ test_focused = _is_test_query(query)
+
+ def key(item: dict) -> tuple:
+ pt = item.get("page_type")
+ is_decision = (not why) and pt == "decision_record"
+ is_test = (not test_focused) and _is_test_page(item)
+ return (1 if (is_decision or is_test) else 0, -(item.get("relevance_score") or 0.0))
+
+ output.sort(key=key)
+
+
+def _norm_decision_title(title: str) -> str:
+ """Normalize a decision title to collapse near-duplicate phrasings."""
+ return re.sub(r"[^a-z0-9]+", " ", (title or "").lower()).strip()
+
+
+def _dedup_decisions(output: list[dict]) -> list[dict]:
+ """Collapse near-duplicate decision records by normalized title (order-preserving).
+
+ Callers dedup AFTER the score sort, so the surviving variant is the
+ highest-scored one; non-decision pages always pass through untouched.
+ """
+ seen: set[str] = set()
+ out: list[dict] = []
+ for item in output:
+ if item.get("page_type") == "decision_record":
+ key = _norm_decision_title(item.get("title", ""))
+ if key and key in seen:
+ continue
+ seen.add(key)
+ out.append(item)
+ return out
+
+
+async def _non_decision_fallback(ctx, query: str, fetch_limit: int) -> list[dict]:
+ """Wider re-fetch keeping only non-decision pages.
+
+ Guard for the window-saturation failure: when the entire over-fetched
+ window is decision records, demotion has nothing to promote and a
+ non-why query returns zero implementation pages. Fetch 4x wider, drop
+ decisions, and let the caller merge the survivors.
+ """
+ results = []
+ src = "vector"
+ # Skipped on a keyless index, which then falls through to the FTS branch.
+ if _pkg.store_has_semantic_vectors(ctx.vector_store):
+ try:
+ results = await asyncio.wait_for(
+ ctx.vector_store.search(query, limit=fetch_limit * 4),
+ timeout=vector_search_timeout_s(),
+ )
+ except TimeoutError:
+ _log.warning("Vector re-fetch timed out; falling back to full-text")
+ except Exception:
+ _log.debug("Vector re-fetch failed; falling back to full-text", exc_info=True)
+ if not results:
+ src = "fts"
+ with contextlib.suppress(Exception):
+ results = await ctx.fts.search(query, limit=fetch_limit * 4)
+
+ out = []
+ for r in results:
+ if r.page_type == "decision_record":
+ continue
+ if r.score < _MIN_RELEVANCE_SCORE:
+ continue
+ out.append(
+ {
+ "page_id": r.page_id,
+ "title": r.title,
+ "page_type": r.page_type,
+ "snippet": r.snippet,
+ "relevance_score": r.score,
+ "sources": [src],
+ }
+ )
+ return out
+
+
+async def _rescue_all_decision_window(
+ ctx, output: list[dict], query: str, fetch_limit: int
+) -> list[dict]:
+ """Merge non-decision pages into an all-decision result window.
+
+ No-op unless the query is non-why AND every current hit is a decision
+ record. Deduplicates by page_id.
+ """
+ if not output or _is_why_shaped(query):
+ return output
+ if not all(item.get("page_type") == "decision_record" for item in output):
+ return output
+ fallback = await _non_decision_fallback(ctx, query, fetch_limit)
+ seen = {item["page_id"] for item in output}
+ output.extend(item for item in fallback if item["page_id"] not in seen)
+ return output
+
+
+# Slots at the tail of a concept window reserved for files reached through the
+# symbol index rather than through their page. A generated file page renders
+# only public symbols, so a question about a private helper or a local name has
+# nothing to match in either the full-text index or the embedding. The symbol
+# index is where those names live. One slot in three, taken from the weakest
+# end of the window, is a cheap price for reaching a class of file the page
+# retrievers structurally cannot see. No-op when the symbol leg finds nothing.
+_SYMBOL_TAIL_DIVISOR = 3
+
+
+def _symbol_tail_reserve(limit: int) -> int:
+ """How many tail slots the symbol leg may take from a window of *limit*."""
+ return limit // _SYMBOL_TAIL_DIVISOR
+
+
+def _reserve_symbol_tail(output: list[dict], limit: int, kind: str | None) -> list[dict]:
+ """Give the weakest window slots to symbol-leg files ranked past ``limit``.
+
+ Concept hits keep the head. Files the symbol leg reached that fusion left
+ outside the window take up to :func:`_symbol_tail_reserve` slots, and the
+ concept hits they displaced fall in behind them (nothing is dropped).
+ """
+ reserve = _symbol_tail_reserve(limit)
+ if reserve <= 0 or kind in ("config", "doc"):
+ # config/doc windows are asking for pages with no symbols in them.
+ return output
+ extra = [item for item in output[limit:] if "symbol" in item.get("sources", ())][:reserve]
+ if not extra:
+ return output
+ head = output[: limit - len(extra)]
+ taken = {id(item) for item in head + extra}
+ return head + extra + [item for item in output if id(item) not in taken]
+
+
+# Path-prefix heuristics for the ``kind`` filter. We classify a hit's
+# target_path against these prefixes; if none match, the hit falls into
+# ``other`` and is dropped only when the caller asked for a specific kind.
+# Tests are not in this list: they come from ``repowise.core.test_paths``, the
+# same rules that stamped the file's ``is_test`` flag at ingestion.
+_CONFIG_PATH_TOKENS = (
+ "pyproject.toml",
+ "package.json",
+ "tsconfig",
+ "setup.py",
+ "setup.cfg",
+ "/.github/",
+ "dockerfile",
+ ".yml",
+ ".yaml",
+ ".toml",
+ ".ini",
+ ".cfg",
+ "lockfile",
+ "package-lock",
+ "uv.lock",
+ "poetry.lock",
+)
+
+
+def _classify_hit_kind(target_path: str, page_type: str) -> str:
+ """Bucket a hit into implementation / test / config / doc.
+
+ Pages without a file behind them (decision records, repo overviews,
+ onboarding pages — all have empty ``target_path``) are docs: letting
+ them fall through to the path heuristics classified every decision
+ record as "implementation", so ``kind="implementation"`` returned
+ decision pages instead of filtering them out.
+
+ ``test`` is the union of tests and test support here: someone asking for
+ ``kind="implementation"`` does not want a ``conftest.py`` back, and someone
+ asking for ``kind="test"`` does.
+
+ Config wins over test, which the token list this replaced never had to
+ decide: ``.github/workflows/tests.yml`` is a workflow whatever it is named,
+ and dropping it from ``kind="config"`` would be the more surprising answer.
+ """
+ tp = (target_path or "").lower()
+ if page_type in ("module_page", "symbol_spotlight") or tp.endswith(".md"):
+ return "doc"
+ if not tp or page_type not in FILE_ROW_TYPES:
+ return "doc"
+ if any(tok in tp for tok in _CONFIG_PATH_TOKENS):
+ return "config"
+ # Original case, not ``tp``: the camel (FooTest.java) and .NET project-dir
+ # (Foo.Tests/) rules are deliberately case-sensitive.
+ if is_test_related_path(target_path):
+ return "test"
+ return "implementation"
+
+
+# Every value :func:`_classify_hit_kind` can return; anything else can only ever
+# match nothing, which is why an unknown kind is dropped rather than filtered on.
+_VALID_KINDS = frozenset({"implementation", "test", "config", "doc"})
+
+
+def _filter_by_kind(output: list[dict], kind: str | None) -> list[dict]:
+ """Keep only hits classified as ``kind`` (no-op when ``kind`` is falsy).
+
+ Runs on the over-fetched list BEFORE the limit cut so the caller still
+ gets up to ``limit`` results of the requested kind.
+ """
+ if not kind:
+ return output
+ return [
+ item
+ for item in output
+ if _classify_hit_kind(item.get("target_path", ""), item.get("page_type", "")) == kind
+ ]
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py b/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py
new file mode 100644
index 000000000..62bb71207
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/retrieval.py
@@ -0,0 +1,307 @@
+"""Fused page retrieval for one repo and across a workspace."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from typing import Any
+
+from repowise.core.persistence.database import get_session
+
+# Patchable names are read off the package at call time so patches reach them.
+from repowise.server.mcp_server import tool_search as _pkg
+from repowise.server.mcp_server._answer_pipeline import (
+ _FILENAME_LEG_RRF_K,
+ _RRF_K,
+ _RRF_SCORE_SCALE,
+ _SYMBOL_LEG_RRF_K,
+ leg_ranks,
+)
+from repowise.server.mcp_server._federation import Federation, merge_ranked
+from repowise.server.mcp_server._helpers import (
+ _VECTOR_TIMEOUT_ENV,
+ _get_repo,
+ _resolve_repo_context,
+ filter_dicts_by_key,
+ vector_search_timeout_s,
+)
+from repowise.server.mcp_server._meta import build_meta as _build_meta
+from repowise.server.mcp_server._page_paths import (
+ PAGELESS_FILE,
+ file_candidates,
+ file_path_of,
+ pageless_path,
+)
+from repowise.server.mcp_server._query_shape import (
+ _MIN_RELEVANCE_SCORE,
+ _fetch_limit_for,
+)
+from repowise.server.mcp_server._retrieval_rank import boost_named_paths
+from repowise.server.mcp_server.tool_search_symbols import file_path_index
+
+from .ordering import _collapse_by_file
+from .ranking import (
+ _dedup_decisions,
+ _downweight_decisions,
+ _downweight_test_pages,
+ _filter_by_kind,
+ _rescue_all_decision_window,
+ _sort_demoting_noise,
+)
+from .serve import (
+ _assign_confidence,
+ _attach_paths,
+ _load_page_info,
+ _page_language,
+ _serve_snippets,
+ _slim_served_rows,
+)
+
+_log = logging.getLogger("repowise.mcp.search")
+
+
+async def _wait_for_vector_store(ctx) -> None:
+ """Block (bounded) until the vector store signals readiness, if it tracks it."""
+ if ctx.vector_store_ready is not None:
+ with contextlib.suppress(TimeoutError):
+ await asyncio.wait_for(ctx.vector_store_ready.wait(), timeout=30.0)
+
+
+async def _safe_fts(ctx, query: str, limit: int) -> list:
+ """FTS search, bounded and failure-swallowing (returns [] on error/timeout)."""
+ if ctx.fts is None:
+ return []
+ with contextlib.suppress(Exception):
+ return await asyncio.wait_for(ctx.fts.search(query, limit=limit), timeout=5.0)
+ return []
+
+
+async def _safe_vector(ctx, query: str, limit: int) -> list:
+ """Vector search, bounded and failure-swallowing (returns [] on error/timeout).
+
+ Readiness is awaited by the caller (``_wait_for_vector_store``) before the
+ fused retrieve runs, so this path does not re-wait.
+
+ Returns nothing on a keyless index. ``store_has_semantic_vectors`` explains
+ why the mock's vectors cannot be ranked on; the reason this is enforced here
+ rather than in ``_fused_retrieve`` is that this function is the only place
+ the vector leg is entered, so a later caller inherits the guard.
+ """
+ if ctx.vector_store is None or not _pkg.store_has_semantic_vectors(ctx.vector_store):
+ return []
+ try:
+ return await asyncio.wait_for(
+ ctx.vector_store.search(query, limit=limit), timeout=vector_search_timeout_s()
+ )
+ except TimeoutError:
+ # Logged, not silent: a timeout drops the whole semantic leg and the
+ # full-text-only hits that remain look like a normal result set.
+ _log.warning(
+ "Vector search exceeded its %gs budget; returning no semantic hits. "
+ "Raise it with %s=<seconds>.",
+ vector_search_timeout_s(),
+ _VECTOR_TIMEOUT_ENV,
+ )
+ except Exception:
+ _log.debug("Vector search failed; returning no semantic hits", exc_info=True)
+ return []
+
+
+def _symbol_leg_target(fused: dict[str, dict], page_id: str) -> str:
+ """The fused page a symbol-leg file page credits: the best page of its file.
+
+ The leg names files, while the page legs may have reached the same file
+ through a symbol page. Crediting only the file page id would leave that
+ hit unlifted and add a weak twin that collapsing then drops.
+ """
+ path = page_id.partition(":")[2]
+ best = None
+ for pid, entry in fused.items():
+ if file_path_of(entry["page_type"], pid.partition(":")[2]) != path:
+ continue
+ if best is None or entry["_rrf"] > fused[best]["_rrf"]:
+ best = pid
+ return best or page_id
+
+
+def _fused_entry(r) -> dict:
+ """Seed a fused-result dict from a retriever hit (RRF score added by caller)."""
+ entry = {
+ "page_id": r.page_id,
+ "title": r.title,
+ "page_type": r.page_type,
+ "snippet": r.snippet,
+ "_rrf": 0.0,
+ "_sources": set(),
+ }
+ if r.page_type == PAGELESS_FILE:
+ # No page row to load it from later.
+ entry["target_path"] = pageless_path(r.page_id)
+ return entry
+
+
+async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | None) -> list[dict]:
+ """Retrieve by fusing FTS, vector and symbol search via Reciprocal Rank Fusion.
+
+ The retrievers run in parallel; a hit's score is the sum of
+ ``1/(rank + k)`` over the retrievers that surfaced it, scaled into the
+ BM25 range the downstream relevance gates were tuned against. Each hit
+ carries a ``sources`` list (``"fts"``, ``"vector"``, ``"symbol"``,
+ ``"filename"``): a page found by several retrievers is a stronger match
+ than one found by one mode. The symbol and filename legs also name files
+ that have no page, as ``page_type: "file"`` rows.
+
+ This mirrors get_answer's ``hybrid_retrieve`` so the two entry points rank
+ the same corpus identically. The prior path ran vector search and fell
+ back to FTS only when vector returned nothing, so a page FTS ranked highly
+ but vector missed never surfaced here.
+
+ A hit is admitted only when its raw retriever score clears
+ ``_MIN_RELEVANCE_SCORE`` — the fused score is rank-based and would
+ otherwise sit well above the floor for every window hit, letting an
+ off-topic query return its nearest-but-unrelated neighbours. Rank position
+ (for RRF) is the hit's place in the retriever's own list, unchanged by the
+ floor.
+ """
+ fts_results, vec_results, sym_results, name_results = await asyncio.gather(
+ _pkg._safe_fts(ctx, query, fetch_limit),
+ _pkg._safe_vector(ctx, query, fetch_limit),
+ _pkg._safe_symbol_search(ctx, query, pageless=True),
+ _pkg._safe_filename_search(ctx, query),
+ )
+
+ fused: dict[str, dict] = {}
+ for rank, r in enumerate(vec_results):
+ if r.score < _MIN_RELEVANCE_SCORE:
+ continue
+ entry = fused.setdefault(r.page_id, _fused_entry(r))
+ entry["_rrf"] += 1.0 / (rank + _RRF_K)
+ entry["_sources"].add("vector")
+ for rank, r in enumerate(fts_results):
+ if r.score < _MIN_RELEVANCE_SCORE:
+ continue
+ entry = fused.setdefault(r.page_id, _fused_entry(r))
+ entry["_rrf"] += 1.0 / (rank + _RRF_K)
+ entry["_sources"].add("fts")
+ # Files whose symbol names the query's words spell, paged or not. Weighted
+ # below the page legs, as in get_answer; it has no raw score to floor.
+ for rank, r in leg_ranks(sym_results):
+ entry = fused.setdefault(_symbol_leg_target(fused, r.page_id), _fused_entry(r))
+ entry["_rrf"] += 1.0 / (rank + _SYMBOL_LEG_RRF_K)
+ entry["_sources"].add("symbol")
+ # Files whose file name the query's words spell, weighted as a name match.
+ # It adds files the other legs missed and never reorders a page they found.
+ # A name of three or more words the query spells in full weighs like a page hit.
+ for rank, (r, full) in leg_ranks(name_results):
+ target = _symbol_leg_target(fused, r.page_id)
+ if target in fused and r.page_type != PAGELESS_FILE:
+ continue
+ entry = fused.setdefault(target, _fused_entry(r))
+ entry["_rrf"] += 1.0 / (rank + (_RRF_K if full else _FILENAME_LEG_RRF_K))
+ entry["_sources"].add("filename")
+
+ output: list[dict] = []
+ for entry in fused.values():
+ if page_type and entry["page_type"] != page_type:
+ continue
+ entry["relevance_score"] = round(entry.pop("_rrf") * _RRF_SCORE_SCALE, 4)
+ entry["sources"] = sorted(entry.pop("_sources"))
+ output.append(entry)
+ output.sort(key=lambda item: item["relevance_score"], reverse=True)
+ return output
+
+
+async def _search_single_repo(
+ ctx, query: str, limit: int, page_type: str | None, kind: str | None = None
+) -> list[dict]:
+ """Run search against a single repo context.
+
+ Fuses FTS and vector retrieval via RRF (see ``_fused_retrieve``), so each
+ hit carries a ``sources`` list naming the retrievers that surfaced it
+ rather than a single backend label.
+
+ The ``kind`` filter runs here, on the over-fetched list and BEFORE the
+ limit cut — filtering after per-repo truncation returned fewer than
+ ``limit`` results (frequently zero) in the federated path.
+ """
+ await _pkg._wait_for_vector_store(ctx)
+
+ fetch_limit = _fetch_limit_for(limit, kind)
+ output = await _fused_retrieve(ctx, query, fetch_limit, page_type)
+
+ _downweight_decisions(output, query)
+ output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)
+
+ # Attach target_path and drop excluded hits per repo (so federated search
+ # honours each repo's own exclude_patterns) before ranking. Runs on the
+ # over-fetched list so the kind filter below has real headroom. Load
+ # precedes the sort/demotion so they see the page metadata they key on
+ # (test demotion needs target_path to classify a test file page).
+ if output:
+ async with get_session(ctx.session_factory) as session:
+ page_info, tombstoned, _ = await _load_page_info(session, output)
+ path_index = await file_path_index(session, (await _get_repo(session)).id)
+ output = [item for item in output if item["page_id"] not in tombstoned]
+ _attach_paths(output, page_info)
+ output = filter_dicts_by_key(output, "target_path", _pkg._get_exclude_spec(ctx.path))
+ boost_named_paths(
+ output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"
+ )
+
+ _downweight_test_pages(output, query)
+ _sort_demoting_noise(output, query)
+ output = _dedup_decisions(output)
+
+ output = _filter_by_kind(output, kind)
+ return _collapse_by_file(output)[:limit]
+
+
+def _neg_score(key: str):
+ """Best-first sort key on a row's *key* score, for :func:`merge_ranked`."""
+ return lambda item: -(item.get(key) or 0.0)
+
+
+async def _federated_search(
+ query: str, limit: int, page_type: str | None, kind: str | None, fed: Federation
+) -> dict:
+ """Search across all repos using Reciprocal Rank Fusion."""
+
+ async def _one_repo(ctx) -> list[dict]:
+ repo_results = await _search_single_repo(ctx, query, limit, page_type, kind)
+ for rank, item in enumerate(repo_results):
+ item["repo"] = ctx.alias
+ item["rrf_score"] = 1.0 / (rank + 60) # RRF constant k=60
+ return repo_results
+
+ answered = await fed.run(_one_repo)
+ contexts = [ctx for ctx, _ in answered]
+ all_results = merge_ranked(
+ (rows for _, rows in answered), _neg_score("rrf_score"), per_repo_cap=limit
+ )
+ output = all_results[:limit]
+
+ # Derive confidence from RRF position
+ _assign_confidence(output, "rrf_score", "confidence_score")
+
+ # No single indexed commit to compare a live HEAD against: per-repo
+ # freshness rides on ``repos_stale`` and the rows' ``stale`` flag instead.
+ response: dict = {"results": output, "_meta": _build_meta()}
+ fed.annotate(response, output)
+ # Drawn from every repo's ranked list, not the merged cut: a workspace
+ # search that spends its window on module pages should still be able to
+ # name files.
+ if candidates := file_candidates(all_results, limit=limit):
+ response["candidates"] = candidates
+ # Last, so nothing above has to know the field is on its way out.
+ _slim_served_rows(output, {ctx.alias: _page_language(ctx) for ctx in contexts})
+ _serve_snippets(output)
+ return response
+
+
+async def _per_repo(repo: str | None, fed: Federation | None, fn) -> list[tuple[Any, Any]]:
+ """``(context, fn(context))`` for the one repo asked for, or every repo via *fed*."""
+ if fed is not None:
+ return await fed.run(fn)
+ ctx = await _resolve_repo_context(repo)
+ return [(ctx, await fn(ctx))]
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/serve.py b/packages/server/src/repowise/server/mcp_server/tool_search/serve.py
new file mode 100644
index 000000000..ce05b5250
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/serve.py
@@ -0,0 +1,275 @@
+"""Served-row shaping: paths, freshness, slimming and freshness claims."""
+
+from __future__ import annotations
+
+from collections.abc import Container
+
+from sqlalchemy import select
+
+from repowise.core.generation.structural_labels import is_structural_title
+from repowise.core.persistence.models import (
+ GitMetadata,
+ Page,
+)
+
+# Patchable names are read off the package at call time so patches reach them.
+from repowise.server.mcp_server import tool_search as _pkg
+from repowise.server.mcp_server._budget import (
+ OmissionCollector,
+ register_post_enforce,
+ register_post_shed,
+)
+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT
+from repowise.server.mcp_server._page_paths import (
+ PAGELESS_FILE,
+ add_row_paths,
+ hit_file_path,
+)
+from repowise.server.mcp_server._query_shape import (
+ _embedded_identifiers,
+ _looks_like_exact_token,
+)
+from repowise.server.mcp_server._references import path_identity, symbol_identity
+
+# Freshness tie-breaker, added (not multiplied) to a hit's fused relevance.
+# The RRF-fused score spaces adjacent ranks ~0.05 apart, so this stays under
+# one rank step: recency orders otherwise-comparable hits without overriding
+# retrieval relevance. Recency itself scales it (0.5 for 90-day activity, 1.0
+# for 30-day).
+_FRESHNESS_TIEBREAK = 0.03
+
+
+def _attach_paths(output: list[dict], page_info: dict) -> None:
+ """Stamp canonical file paths and split symbol ids into their own field.
+
+ A ``symbol_spotlight``'s target_path is ``file.py::Symbol``: a page id, and
+ not a file path. Preserve it as ``symbol_id`` for ``get_symbol`` and expose
+ only the file portion through ``target_path`` / ``file``. Every search
+ branch calls this one function so they agree on the row's location.
+ """
+ for item in output:
+ item["target_path"] = page_info.get(item["page_id"], item.get("target_path", ""))
+ if "::" in item["target_path"]:
+ item["symbol_id"] = symbol_identity(item["target_path"])
+ item["target_path"] = path_identity(item["target_path"].split("::", 1)[0])
+ item["file"] = item["target_path"]
+
+
+def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:
+ """Strip ``page_id`` from every result that can rebuild it, in place.
+
+ A page id is ``compute_page_id(page_type, target_path)`` — literally
+ ``f"{page_type}:{target_path}"`` (``core/generation/models.py``) — and both
+ halves ship in the same row, so the field restates two of its siblings. It
+ is ranking plumbing: internal uses key the fused dict on it, and none of
+ that needs to reach the wire.
+
+ Conditional rather than unconditional, and the condition is the derivation
+ itself. ``_attach_paths`` writes ``target_path: ""`` for a hit whose Page
+ row did not load, and a row like that cannot be rebuilt — so it keeps its
+ id instead of losing one. The rule is "drop it only where it is provably
+ redundant", which is also the check: if this ever stops firing, the
+ invariant it rests on has changed.
+
+ Consumers rebuild with the same expression (``ui/src/chat/source-citations``).
+
+ ``add_row_paths`` runs first and deletes ``target_path`` wherever it equals
+ the new ``path``, so the id's second half is ``target_path`` when present
+ and ``path`` otherwise. Without that fallback an ordinary file row rebuilt
+ as ``"file_page:"`` and kept its id.
+ """
+ for item in results:
+ # A symbol page's id ends in ``::Symbol`` and ``_attach_paths`` moved that
+ # half into ``symbol_id``, so ``target_path`` is only its file.
+ target = (
+ item.get("symbol_id")
+ if item.get("page_type") == "symbol_spotlight"
+ else None
+ ) or item.get("target_path") or item.get("path", "")
+ derived = f"{item.get('page_type', '')}:{target}"
+ # A row for a file with no page has an id that names no page.
+ if item.get("page_id") == derived or item.get("page_type") == PAGELESS_FILE:
+ item.pop("page_id", None)
+ return results
+
+
+def _page_language(ctx) -> str:
+ """The language *ctx*'s structural page titles were generated in."""
+ try:
+ from repowise.core.repo_config import load_repo_config
+
+ return str(load_repo_config(ctx.path).get("language") or "en")
+ except Exception:
+ return "en"
+
+
+def _slim_served_rows(
+ rows: list[dict], languages: dict[str | None, str] | None = None
+) -> list[dict]:
+ """The last step before a search reply leaves: one location per row.
+
+ ``path`` (plus ``symbol_id`` on symbol rows) is the row's only location, so
+ ``file``, a ``page_id`` that rebuilds from them, and a ``title`` that is just
+ the structural label wrapped around them are all dropped. Each is dropped
+ only where the caller can provably rebuild it. ``sources: ["fts"]`` goes when
+ the reply's own ``_meta.semantic_search`` already says retrieval is
+ full-text-only; an index with embeddings is unchanged.
+
+ *languages* maps a row's ``repo`` alias (``None`` for a single-repo reply) to
+ the language its titles were generated in.
+ """
+ add_row_paths(rows)
+ _drop_derivable_page_ids(rows)
+ fts_only = _pkg.semantic_search_state() is False
+ for row in rows:
+ target = row.get("symbol_id") or row.get("path")
+ if target and is_structural_title(
+ (languages or {}).get(row.get("repo"), "en"),
+ row.get("page_type", ""),
+ target,
+ row.get("title"),
+ ):
+ row.pop("title", None)
+ if fts_only and row.get("sources") == ["fts"]:
+ row.pop("sources", None)
+ return rows
+
+
+def _serve_snippets(results: list[dict]) -> None:
+ """Drop each snippet's leading page heading, after ranking has read it."""
+ for item in results:
+ if item.get("snippet"):
+ item["snippet"] = _pkg.strip_leading_headings(item["snippet"])
+
+
+def _drop_internal_ranking_fields(results: list[dict]) -> None:
+ """Keep calibration diagnostics internal to the ranking pipeline."""
+ for item in results:
+ item.pop("_coverage", None)
+ item.pop("_coverage_multiplier", None)
+ item.pop("_confidence_score_factor", None)
+ item.pop("_raw_score", None)
+
+
+async def _load_page_info(
+ session, output: list[dict], *, with_git: bool = False
+) -> tuple[dict, set, dict]:
+ """Batch-load target paths, tombstones, and git.
+
+ Returns ``(page_info, tombstoned, git_map)`` where ``page_info`` maps
+ page_id -> target_path, ``tombstoned`` is the set of tombstoned page_ids,
+ and ``git_map`` maps file_path -> GitMetadata (empty unless ``with_git``).
+ """
+ page_ids = [item["page_id"] for item in output]
+ res = await session.execute(
+ select(Page.id, Page.target_path, Page.freshness_status).where(Page.id.in_(page_ids))
+ )
+ rows = res.all()
+ page_info = {row[0]: row[1] for row in rows}
+ tombstoned = {row[0] for row in rows if row[2] == "tombstone"}
+
+ git_map: dict[str, GitMetadata] = {}
+ if with_git:
+ target_paths = [tp for tp in page_info.values() if tp]
+ if target_paths:
+ git_res = await session.execute(
+ select(GitMetadata).where(GitMetadata.file_path.in_(target_paths))
+ )
+ git_map = {g.file_path: g for g in git_res.scalars().all()}
+ return page_info, tombstoned, git_map
+
+
+def _apply_freshness_boost(item: dict, gm: GitMetadata | None) -> None:
+ """Nudge a recently-active file up as a bounded tie-breaker (in place).
+
+ Additive, not multiplicative: the fused relevance score is RRF-based, which
+ compresses adjacent ranks to roughly ``_FRESHNESS_TIEBREAK`` apart. A
+ percentage boost on that scale would let a fresh rank-2 hit leapfrog a
+ stale rank-0 hit, overriding retrieval relevance with recency. Sized below
+ one rank step, freshness only orders hits retrieval already ranks close
+ together.
+ """
+ if not gm or not item.get("relevance_score"):
+ return
+ c30 = gm.commit_count_30d or 0
+ c90 = gm.commit_count_90d or 0
+ if c30 > 0:
+ recency = 1.0
+ elif c90 > 0:
+ recency = 0.5
+ else:
+ recency = 0.0
+ item["relevance_score"] = round(item["relevance_score"] + _FRESHNESS_TIEBREAK * recency, 4)
+
+
+def _assign_confidence(output: list[dict], score_key: str, target_key: str) -> None:
+ """Derive ``target_key`` for each item from its ``score_key`` position (in place)."""
+ if not output:
+ return
+ max_score = max((item.get(score_key) or 0) for item in output)
+ for item in output:
+ raw = item.get(score_key) or 0
+ item[target_key] = round(raw / max_score, 2) if max_score > 0 else 0.0
+
+
+def _refresh_served_targets(response: dict, _collector: OmissionCollector | None) -> None:
+ """Re-derive target-scoped freshness from the hits that survived shedding.
+
+ ``_meta.targets`` claims which files the response served. Shedding decides
+ that, so this has to run after it or the claim names rows the caller never
+ received.
+ """
+ if response.get("truncated") and isinstance(response.get("_meta"), dict):
+ response["_meta"]["targets"] = _result_paths(response.get("results") or [])
+
+
+def _refresh_served_targets_final(response: dict) -> None:
+ """Same claim, re-derived after the final size guard may have cut again."""
+ _refresh_served_targets(response, None)
+
+
+register_post_shed("search_codebase", _refresh_served_targets)
+register_post_enforce("search_codebase", _refresh_served_targets_final)
+
+
+def _result_paths(results: list[dict]) -> list[str]:
+ """File paths a result set serves, for target-scoped freshness.
+
+ Symbol hits carry ``file``; page hits are resolved through their page type,
+ so a module page's group key and an onboarding slot resolve to nothing
+ rather than being handed on as if they were files. Freshness is per-file
+ git metadata, and a directory has none, so a page id reaching here could
+ only ever fail to match.
+
+ An empty result set returns ``[]`` (meaning "no file content served",
+ which suppresses the repo-level stale warning by design).
+ """
+ paths: list[str] = []
+ for item in results:
+ p = hit_file_path(item)
+ if p:
+ paths.append(p)
+ return paths
+
+
+def _grep_hint_for(query: str, names: Container[str] | None = None) -> str | None:
+ """Zero-result recovery hint for identifier-shaped queries, else ``None``.
+
+ Only attached when the search produced nothing (see call sites) — a
+ populated result set needs no escape hatch. Points back into the tool
+ surface first; Grep is named only for the one job it genuinely wins:
+ an exhaustive literal sweep of every usage (e.g. before a rename).
+ """
+ if _looks_like_exact_token(query):
+ return (
+ f"No indexed match for identifier {query!r}. Retry with "
+ 'mode="symbol" (or check spelling/casing). ' + EXHAUSTIVE_SWEEP_HINT
+ )
+ if idents := _embedded_identifiers(query, names):
+ shown = ", ".join(repr(t) for t in idents[:3])
+ return (
+ f"Query names identifier(s) {shown} but nothing matched. Search "
+ 'the identifier alone with mode="symbol", then pipe the hit '
+ "into get_symbol for its body. " + EXHAUSTIVE_SWEEP_HINT
+ )
+ return None
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/structured.py b/packages/server/src/repowise/server/mcp_server/tool_search/structured.py
new file mode 100644
index 000000000..c17b8b148
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/structured.py
@@ -0,0 +1,395 @@
+"""Symbol, path and hybrid search across one repo or a federation."""
+
+from __future__ import annotations
+
+from collections.abc import Container
+from dataclasses import dataclass
+from typing import Any
+
+from repowise.core.persistence.database import get_session
+
+# Patchable names are read off the package at call time so patches reach them.
+from repowise.server.mcp_server import tool_search as _pkg
+from repowise.server.mcp_server._federation import Federation, merge_ranked
+from repowise.server.mcp_server._helpers import _get_repo
+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+from repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINT
+from repowise.server.mcp_server._meta import build_meta as _build_meta
+from repowise.server.mcp_server._page_paths import (
+ file_candidates,
+ hit_file_path,
+)
+from repowise.server.mcp_server._query_shape import (
+ NOT_THE_NAMED_SYMBOL,
+ _canonical_symbol_query,
+ _embedded_identifiers,
+ _has_exact_symbol,
+ _identifier_candidates,
+ _looks_like_code_name,
+ _mark_not_the_named_symbol,
+ _names_a_path,
+)
+from repowise.server.mcp_server._query_terms import content_terms
+from repowise.server.mcp_server.tool_search_symbols import (
+ _MAX_CANDIDATES,
+ IssueFiles,
+ file_path_index,
+ search_paths_single,
+)
+
+from .ordering import (
+ _FILE_WINDOW_OVERFETCH,
+ _collapse_by_file,
+ _hybrid_window,
+ _protect_exact_paths,
+ _protect_exact_symbols,
+ _protect_named_symbols,
+)
+from .retrieval import _neg_score, _per_repo, _search_single_repo
+from .serve import _page_language, _result_paths, _serve_snippets, _slim_served_rows
+
+
+def _lead_candidates(response: dict, issue: IssueFiles | None, limit: int) -> None:
+ """Trace files lead ``candidates``. A named identifier's defining file only
+ fills a free slot after the ranked ones: ranked above them, it displaces
+ better hits on ordinary questions. Capped at ``limit``."""
+ if issue is None or not (issue.traced or issue.named):
+ return
+ ranked = [c["path"] for c in response.get("candidates") or []]
+ paths = list(dict.fromkeys(issue.traced + ranked + issue.named))
+ response["candidates"] = [{"path": p} for p in paths[:limit]]
+
+
+def _tag_repo(items: list[dict], ctx, multi: bool) -> None:
+ if multi:
+ for item in items:
+ item["repo"] = ctx.alias
+
+
+def _missing_named_symbols(
+ candidates: list[str], symbols: list[dict], canonical: bool, concepts: list[dict]
+) -> list[str]:
+ """Code-shaped names the query asks after that no returned symbol matches
+ exactly (or returned module path carries): they do not exist here. Judged
+ per name, so an indexed name beside a missing one hides nothing. Their
+ fuzzy neighbours would stand in for them, so the caller keeps only exact
+ symbols; the pages are marked here as related to the question, not the
+ symbol."""
+ if canonical:
+ return []
+ page_paths = [c.get("target_path") or "" for c in concepts]
+ missing = [
+ c
+ for c in candidates
+ if _looks_like_code_name(c)
+ and not _has_exact_symbol([c], symbols)
+ and not _names_a_path(c, page_paths)
+ ]
+ if missing:
+ _mark_not_the_named_symbol(concepts)
+ return missing
+
+
+@dataclass(frozen=True)
+class _SearchPlan:
+ """One structured search: the caller's arguments plus the derived query."""
+
+ query: str
+ limit: int
+ page_type: str | None
+ kind: str | None
+ symbol_kind: str | None
+ mode: str
+ names: Container[str] | None
+ canonical_symbol: tuple[Any, str] | None
+ symbol_query: str
+ fetch: int
+ multi: bool
+
+
+@dataclass
+class _Pools:
+ """Each row kind merged and ranked across the repos searched."""
+
+ contexts: list[Any]
+ symbols: list[dict]
+ spare: list[dict]
+ files: list[dict]
+ indexed_paths: list[str]
+ concepts: list[dict]
+
+
+def _symbol_query_for(
+ query: str, mode: str, names: Container[str] | None, canonical_symbol: Any
+) -> str:
+ """The text the symbol scorer ranks on."""
+ # Scored on the identifiers, not the prose: words like "is" would match
+ # ``is_ci`` and bury the symbol the question names.
+ symbol_query = query
+ if canonical_symbol:
+ symbol_query = canonical_symbol[1]
+ if mode == "hybrid":
+ _idents = _embedded_identifiers(query, names)
+ # No identifier: content words, else the raw text.
+ symbol_query = (
+ " ".join(_idents) if _idents else " ".join(content_terms(query)) or query
+ )
+ return symbol_query
+
+
+def _symbol_fetch(mode: str, limit: int) -> int:
+ """How many symbol rows each repo fetches."""
+ # Hybrid collapses rows per file, so it over-fetches; symbol mode takes every
+ # candidate so ``fuzzy_omitted`` can count what an exact hit hides.
+ fetch = limit * _FILE_WINDOW_OVERFETCH if mode == "hybrid" else limit
+ if mode == "symbol":
+ fetch = _MAX_CANDIDATES
+ return fetch
+
+
+async def _search_one_repo(
+ ctx: Any, plan: _SearchPlan
+) -> tuple[list[dict], list[dict], list[str], list[dict]]:
+ """Symbol, path and page rows for one repo, plus its indexed paths in path mode."""
+ s: list[dict] = []
+ f: list[dict] = []
+ paths: list[str] = []
+ c: list[dict] = []
+ if plan.mode in ("symbol", "hybrid"):
+ s = await _pkg.search_symbols_single(
+ ctx, plan.symbol_query, plan.fetch, symbol_kind=plan.symbol_kind, kind=plan.kind
+ )
+ if plan.mode == "path":
+ f = await search_paths_single(ctx, plan.query, plan.limit)
+ async with get_session(ctx.session_factory) as session:
+ paths = (await file_path_index(session, (await _get_repo(session)).id)).paths
+ if plan.mode == "hybrid":
+ c = await _search_single_repo(ctx, plan.query, plan.limit, plan.page_type, plan.kind)
+ for item in c:
+ item["type"] = "page"
+ await attach_hit_symbols(ctx, plan.query, c)
+ for rows in (s, f, c):
+ _tag_repo(rows, ctx, plan.multi)
+ return s, f, paths, c
+
+
+def _merge_pools(answered: list[tuple[Any, Any]], limit: int) -> _Pools:
+ """Merge each repo's rows into one ranked pool per row kind."""
+ parts = [part for _, part in answered]
+ return _Pools(
+ contexts=[ctx for ctx, _ in answered],
+ symbols=merge_ranked((p[0] for p in parts), _neg_score("score"), per_repo_cap=limit),
+ # Rows past each repo's limit only backfill slots that collapsing frees.
+ spare=merge_ranked((p[0][limit:] for p in parts), _neg_score("score")),
+ files=merge_ranked((p[1] for p in parts), _neg_score("score")),
+ indexed_paths=[path for p in parts for path in p[2]],
+ concepts=merge_ranked((p[3] for p in parts), _neg_score("relevance_score")),
+ )
+
+
+def _protect_exact(plan: _SearchPlan, pools: _Pools, candidates: list[str]) -> None:
+ """Keep exact name and path matches ahead of their fuzzy neighbours."""
+ if plan.mode == "symbol":
+ pools.symbols = _protect_exact_symbols(plan.query, pools.symbols)
+ elif plan.mode == "hybrid" and candidates:
+ pools.symbols = _protect_named_symbols(candidates, pools.symbols)
+ if plan.mode == "path":
+ pools.files = _protect_exact_paths(plan.query, pools.files, pools.indexed_paths)
+
+
+def _symbol_mode_results(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
+) -> tuple[list[dict], int]:
+ """Symbol mode rows, and how many fuzzy names an exact hit hides."""
+ # An exact hit is the answer; its fuzzy neighbours are only counted.
+ fuzzy_omitted = 0
+ pool = pools.symbols + pools.spare
+ named = [s for s in pool if _has_exact_symbol(candidates, [s])] if exact else []
+ if named:
+ canonical_symbol = plan.canonical_symbol
+ ident = (canonical_symbol[1] if canonical_symbol else plan.query).strip().lower()
+ shown = {id(s) for s in named}
+ fuzzy_omitted = sum(
+ 1 for s in pool if id(s) not in shown and ident in (s.get("name") or "").lower()
+ )
+ return (named or pools.symbols)[: plan.limit], fuzzy_omitted
+
+
+def _hybrid_mode_results(plan: _SearchPlan, pools: _Pools, exact: bool) -> list[dict]:
+ """Hybrid rows: symbol matches interleaved with concept pages for new files."""
+ window = _hybrid_window(
+ plan.query, pools.symbols, pools.concepts, plan.limit, exact, plan.names
+ )
+ # One row per file; freed slots fill with the next pages, then symbols.
+ served = {id(item) for item in window}
+ rest = [
+ item for item in pools.concepts + pools.symbols + pools.spare if id(item) not in served
+ ]
+ results = _collapse_by_file(window + rest)[: plan.limit]
+ # Non-file pages leave the window unrefilled unless the caller asked for
+ # pages by type or kind="doc".
+ if not plan.page_type and plan.kind != "doc":
+ results = [item for item in results if hit_file_path(item)]
+ return results
+
+
+def _select_results(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str], exact: bool
+) -> tuple[list[dict], int]:
+ """The served rows for the plan's mode, and the symbol-mode fuzzy count."""
+ if plan.mode == "symbol":
+ return _symbol_mode_results(plan, pools, candidates, exact)
+ if plan.mode == "path":
+ return pools.files[: plan.limit], 0
+ return _hybrid_mode_results(plan, pools, exact), 0
+
+
+async def _single_repository(contexts: list[Any], multi: bool) -> Any:
+ """The repository row a single-repo search ran against, else None."""
+ if multi or not contexts:
+ return None
+ async with get_session(contexts[0].session_factory) as session:
+ return await _get_repo(session)
+
+
+def _exact_match_note(
+ candidates: list[str],
+ exact: bool,
+ missing: list[str],
+ fuzzy_omitted: int,
+ has_results: bool,
+ grep_hint: str | None,
+) -> str | None:
+ """The note qualifying an identifier query's results, if one is owed."""
+ if missing:
+ shown = ", ".join(repr(c) for c in missing[:3])
+ # An empty window's grep_hint already carries the sweep advice.
+ pages = f" Any page here is {NOT_THE_NAMED_SYMBOL}." if has_results else ""
+ sweep = "" if (grep_hint and not has_results) else " " + EXHAUSTIVE_SWEEP_HINT
+ return (
+ f"No indexed symbol is named {shown}, so no symbol is returned "
+ f"for it.{pages} Recheck the spelling, or search a shorter part "
+ f"of the name.{sweep}"
+ )
+ if not exact:
+ shown = ", ".join(repr(c) for c in candidates[:3])
+ return (
+ f"No indexed symbol exactly matches {shown}. The results are "
+ "fuzzy neighbours ranked by token overlap — confirm a hit names "
+ "what you meant before relying on it. If you expected an exact "
+ "symbol, recheck spelling/casing. " + EXHAUSTIVE_SWEEP_HINT
+ )
+ if fuzzy_omitted:
+ return (
+ f"{fuzzy_omitted} other symbols contain this name; search a "
+ "longer or partial name to list them."
+ )
+ return None
+
+
+def _match_named(
+ plan: _SearchPlan, pools: _Pools, candidates: list[str]
+) -> tuple[bool, list[str]]:
+ """Whether a returned symbol names the query exactly, and the names none does.
+
+ Missing names leave only exact symbols in *pools*.
+ """
+ exact = _has_exact_symbol(candidates, pools.symbols) if candidates else False
+ missing = _missing_named_symbols(
+ candidates, pools.symbols, bool(plan.canonical_symbol), pools.concepts
+ )
+ if missing:
+ pools.symbols = [s for s in pools.symbols if _has_exact_symbol(candidates, [s])]
+ pools.spare = [s for s in pools.spare if _has_exact_symbol(candidates, [s])]
+ return exact, missing
+
+
+def _attach_match_notes(
+ response: dict,
+ candidates: list[str],
+ exact: bool,
+ missing: list[str],
+ grep_hint: str | None,
+) -> None:
+ """Exact-match flag, its note, and the grep fallback for an empty window."""
+ results = response["results"]
+ # Fuzzy neighbours of a missing name look authoritative, so say so.
+ if candidates:
+ response["exact_match"] = exact and not missing
+ note = _exact_match_note(
+ candidates,
+ exact,
+ missing,
+ response.get("fuzzy_omitted", 0),
+ bool(results),
+ grep_hint,
+ )
+ if note:
+ response["note"] = note
+ if grep_hint and not results:
+ response["grep_hint"] = grep_hint
+
+
+async def _structured_search(
+ query: str,
+ limit: int,
+ page_type: str | None,
+ kind: str | None,
+ symbol_kind: str | None,
+ repo: str | None,
+ mode: str,
+ grep_hint: str | None,
+ names: Container[str] | None = None,
+ fed: Federation | None = None,
+) -> dict:
+ """Run symbol / path / hybrid search, across repos when federated, and shape the response.
+
+ ``grep_hint`` is attached only when nothing was found.
+ """
+ multi = fed is not None and fed.multi
+ canonical_symbol = _canonical_symbol_query(query)
+ plan = _SearchPlan(
+ query=query,
+ limit=limit,
+ page_type=page_type,
+ kind=kind,
+ symbol_kind=symbol_kind,
+ mode=mode,
+ names=names,
+ canonical_symbol=canonical_symbol,
+ symbol_query=_symbol_query_for(query, mode, names, canonical_symbol),
+ fetch=_symbol_fetch(mode, limit),
+ multi=multi,
+ )
+
+ async def _one_repo(ctx) -> tuple[list[dict], list[dict], list[str], list[dict]]:
+ return await _search_one_repo(ctx, plan)
+
+ pools = _merge_pools(await _per_repo(repo, fed, _one_repo), limit)
+ candidates = _identifier_candidates(query, mode, names)
+ _protect_exact(plan, pools, candidates)
+ exact, missing = _match_named(plan, pools, candidates)
+
+ results, fuzzy_omitted = _select_results(plan, pools, candidates, exact)
+ repository = await _single_repository(pools.contexts, multi)
+
+ response: dict = {
+ "results": results,
+ "mode": mode,
+ "_meta": _build_meta(repository=repository, targets=_result_paths(results)),
+ }
+ if fuzzy_omitted:
+ response["fuzzy_omitted"] = fuzzy_omitted
+ # Not ``candidates``: that name holds the query's identifiers the note quotes.
+ if file_cands := file_candidates(results, limit=limit):
+ response["candidates"] = file_cands
+ _attach_match_notes(response, candidates, exact, missing, grep_hint)
+ if fed is not None:
+ fed.annotate(response, results)
+ # Last: slims fields the steps above still read.
+ _slim_served_rows(
+ results,
+ {(ctx.alias if multi else None): _page_language(ctx) for ctx in pools.contexts},
+ )
+ _serve_snippets(results)
+ return response
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search/tool.py b/packages/server/src/repowise/server/mcp_server/tool_search/tool.py
new file mode 100644
index 000000000..2843b758e
--- /dev/null
+++ b/packages/server/src/repowise/server/mcp_server/tool_search/tool.py
@@ -0,0 +1,282 @@
+"""MCP Tool 5: search_codebase — semantic search over the wiki."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from repowise.core.persistence.database import get_session
+from repowise.core.registry import ToolRecipe
+from repowise.core.registry import mcp_tool_registry as mcp
+
+# Patchable names are read off the package at call time so patches reach them.
+from repowise.server.mcp_server import tool_search as _pkg
+from repowise.server.mcp_server._federation import Federation
+from repowise.server.mcp_server._helpers import (
+ _get_repo,
+ _resolve_repo_context,
+ attach_ignored_arguments,
+ filter_dicts_by_key,
+ resolve_enum_argument,
+)
+from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+from repowise.server.mcp_server._line_hits import attach_line_hits
+from repowise.server.mcp_server._meta import build_meta as _build_meta
+from repowise.server.mcp_server._page_paths import file_candidates
+from repowise.server.mcp_server._query_shape import (
+ _VALID_MODES,
+ _fetch_limit_for,
+ _resolve_mode,
+)
+from repowise.server.mcp_server._retrieval_rank import (
+ boost_named_paths,
+ rerank_pages_first,
+)
+from repowise.server.mcp_server.tool_search_symbols import (
+ IssueFiles,
+ file_path_index,
+ indexed_names,
+ issue_files,
+)
+
+from .ordering import _collapse_by_file
+from .ranking import (
+ _VALID_KINDS,
+ _dedup_decisions,
+ _downweight_decisions,
+ _downweight_test_pages,
+ _filter_by_kind,
+ _rescue_all_decision_window,
+ _reserve_symbol_tail,
+ _sort_demoting_noise,
+)
+from .retrieval import _federated_search, _fused_retrieve, _per_repo
+from .serve import (
+ _apply_freshness_boost,
+ _assign_confidence,
+ _attach_paths,
+ _drop_internal_ranking_fields,
+ _grep_hint_for,
+ _load_page_info,
+ _page_language,
+ _result_paths,
+ _serve_snippets,
+ _slim_served_rows,
+)
+from .structured import _structured_search
+
+_log = logging.getLogger("repowise.mcp.search")
+
+
+def _query_from_pattern(query: str, pattern: str | None, ignored: list[dict[str, Any]]) -> str:
+ """``pattern`` fills an empty ``query``; beside a real one it is reported ignored."""
+ # Hosts that defer tool schemas let a model guess grep's argument name.
+ if pattern is not None and not query.strip():
+ return pattern
+ if pattern is not None:
+ ignored.append(
+ {"argument": "pattern", "values": [pattern], "valid": [], "superseded_by": "query"}
+ )
+ return query
+
+
+def _resolve_kind_and_mode(
+ kind: str | None, mode: str, ignored: list[dict[str, Any]]
+) -> tuple[str | None, str]:
+ """Validate ``kind`` and ``mode``; an unknown value is dropped and reported."""
+ kind = resolve_enum_argument(kind, _VALID_KINDS, argument="kind", ignored=ignored)
+ # Modes are case-insensitive; only a genuine miss is dropped, under its own spelling.
+ if mode is not None and mode.lower() in _VALID_MODES:
+ mode = mode.lower()
+ mode = resolve_enum_argument(mode, _VALID_MODES, argument="mode", ignored=ignored)
+ return kind, mode
+
+
+async def _indexed_names(repo: str | None, fed: Federation | None, query: str) -> set[str]:
+ """Indexed names the query mentions, loaded once so every stage validates alike."""
+ names: set[str] = set()
+ for _, repo_names in await _per_repo(repo, fed, lambda ctx: indexed_names([ctx], query)):
+ names |= repo_names
+ return names
+
+
+async def _issue_files(repo: str | None, query: str, names: set[str]) -> IssueFiles | None:
+ """Files the query traces or names; single repo only (a federation has no one tree)."""
+ if repo == "all":
+ return None
+ try:
+ return await issue_files(await _resolve_repo_context(repo), query, names)
+ except Exception:
+ _log.warning("search_codebase: issue file lookup failed", exc_info=True)
+ return None
+
+
+async def _rank_concept_pages(ctx: Any, output: list[dict], query: str, repo_id: Any) -> list[dict]:
+ """Drop tombstoned and excluded pages, then boost, demote and dedupe the rest."""
+ async with get_session(ctx.session_factory) as session:
+ page_info, tombstoned, git_map = await _load_page_info(session, output, with_git=True)
+ path_index = await file_path_index(session, repo_id)
+ # Tombstoned pages document deleted/renamed files — never results.
+ output = [item for item in output if item["page_id"] not in tombstoned]
+ _attach_paths(output, page_info)
+ output = filter_dicts_by_key(output, "target_path", _pkg._get_exclude_spec(ctx.path))
+ for item in output:
+ target_path = page_info.get(item["page_id"])
+ _apply_freshness_boost(item, git_map.get(target_path) if target_path else None)
+ output = rerank_pages_first(output, query, score_key="relevance_score", floor=0.5)
+ # After the coverage rerank, whose window-relative weights cannot tell
+ # a word rare across the repo from one rare in these few hits.
+ boost_named_paths(
+ output, query, path_index.word_counts, len(path_index.paths), score_key="relevance_score"
+ )
+ _downweight_test_pages(output, query)
+ _sort_demoting_noise(output, query)
+ return _dedup_decisions(output)
+
+
+async def _concept_search(
+ query: str,
+ limit: int,
+ page_type: str | None,
+ kind: str | None,
+ repo: str | None,
+ issue: IssueFiles | None,
+ grep_hint: str | None,
+) -> tuple[dict, Any]:
+ """Single-repo concept search; returns the response and the repo context it read."""
+ ctx = await _resolve_repo_context(repo)
+ async with get_session(ctx.session_factory) as session:
+ repository = await _get_repo(session)
+ await _pkg._wait_for_vector_store(ctx)
+
+ # Over-fetch: post-filters trim hits, and decision down-weighting needs file
+ # pages inside the window to promote.
+ fetch_limit = _fetch_limit_for(limit, kind)
+ output = await _fused_retrieve(ctx, query, fetch_limit, page_type)
+ _downweight_decisions(output, query)
+ output = await _rescue_all_decision_window(ctx, output, query, fetch_limit)
+ if output:
+ output = await _rank_concept_pages(ctx, output, query, repository.id)
+
+ output = _filter_by_kind(output, kind)
+ # A file page and a symbol page of one file are one place to look.
+ output = _collapse_by_file(output)
+ # Files the page retrievers cannot see (a private helper, a local name) get
+ # the weakest tail slots.
+ output = _reserve_symbol_tail(output, limit, kind)
+ _drop_internal_ranking_fields(output)
+ # The full ranked pool, kept so ``candidates`` can reach past the cut.
+ ranked = list(output)
+ output = output[:limit]
+ await attach_hit_symbols(ctx, query, output)
+ _assign_confidence(output, "relevance_score", "confidence_score")
+
+ response: dict = {
+ "results": output,
+ "_meta": _build_meta(repository=repository, targets=_result_paths(output)),
+ }
+ if candidates := file_candidates(ranked, limit=limit):
+ response["candidates"] = candidates
+ _pkg._lead_candidates(response, issue, limit)
+ if grep_hint and not output:
+ response["grep_hint"] = grep_hint
+ return response, ctx
+
+
+ surface_order=40,
+ artifact_type="search_results",
+ presentation="search_results",
+ evidence_basis="inferred",
+ recipes=(
+ ToolRecipe(
+ "find_raw_hits",
+ 'search_codebase(query="identifier or path", mode="auto")',
+ ("search_codebase",),
+ ),
+ ),
+)
+async def search_codebase(
+ query: str = "",
+ limit: int = 5,
+ page_type: str | None = None,
+ kind: str | None = None,
+ repo: str | None = None,
+ mode: str = "auto",
+ symbol_kind: str | None = None,
+ pattern: str | None = None,
+) -> dict:
+ """Find code by concept, symbol, or path — hybrid codebase search.
+
+ For questions (how/where/why), call get_answer instead: it runs this
+ retrieval internally and returns a cited answer. Use this for raw
+ ranked hits: enumerating matches, resolving an identifier to a symbol_id,
+ or scoping get_context.
+
+ mode="auto" routes by query shape: identifiers search symbols (symbol_id,
+ path, line bounds for get_symbol), paths resolve files (for get_context),
+ prose runs wiki-semantic search, and mixed queries run hybrid (symbol hits,
+ then file-backed pages only). Decision records rank below file pages
+ unless the query is why-shaped.
+
+ Rows naming a file carry `path`; concept pages add `symbols`
+ (name:line) the query matches; the top 3 add `matched_lines`
+ (line, text), sample lines sharing its words. `candidates` lists up to `limit`
+ distinct files to Read, best first. Identifier or literal queries also
+ return `lines` (path, line, kind, text; definitions first) read from live
+ files; when `complete` is true they are every match, so no grep is needed.
+
+ Args:
+ query: identifier, path, or natural language.
+ limit: max results (default 5); distinct files outside mode="symbol"
+ (same-file symbols in `symbols`).
+ page_type: file_page (per-file docs) or module_page (subsystem
+ pages); any stored page type filters.
+ kind: implementation | test | config | doc (concept/symbol modes).
+ repo: alias, or "all" for workspace-wide.
+ mode: auto | concept | symbol | path | hybrid.
+ symbol_kind: filter symbol hits (function|class|method|...).
+ pattern: alias for query.
+ """
+ ignored: list[dict[str, Any]] = []
+ query = _query_from_pattern(query, pattern, ignored)
+ if not query.strip():
+ return {
+ "results": [],
+ "error": "search_codebase requires `query` (or its alias `pattern`).",
+ "_meta": _build_meta(),
+ }
+ kind, mode = _resolve_kind_and_mode(kind, mode, ignored)
+ # One fan-out state for the whole call: a repo that fails one stage is
+ # skipped by the next and reported once.
+ fed = Federation() if repo == "all" else None
+ names = await _indexed_names(repo, fed, query)
+ grep_hint = _grep_hint_for(query, names)
+ resolved_mode = _resolve_mode(query, mode, names)
+ issue = await _issue_files(repo, query, names)
+
+ if resolved_mode in ("symbol", "path", "hybrid"):
+ structured = await _structured_search(
+ query, limit, page_type, kind, symbol_kind, repo, resolved_mode, grep_hint, names, fed
+ )
+ _pkg._lead_candidates(structured, issue, limit)
+ await attach_line_hits(structured, query, resolved_mode, names, repo)
+ attach_ignored_arguments(structured, ignored)
+ return structured
+
+ if fed is not None:
+ # kind is filtered per repo before each repo's limit cut, so the fused
+ # list is already kind-pure and full.
+ federated = await _federated_search(query, limit, page_type, kind, fed)
+ if grep_hint and not federated.get("results"):
+ federated["grep_hint"] = grep_hint
+ attach_ignored_arguments(federated, ignored)
+ return federated
+
+ response, ctx = await _concept_search(query, limit, page_type, kind, repo, issue, grep_hint)
+ await attach_line_hits(response, query, resolved_mode, names, repo)
+ attach_ignored_arguments(response, ignored)
+ # Last, so nothing above has to know the field is on its way out.
+ _slim_served_rows(response["results"], {None: _page_language(ctx)})
+ _serve_snippets(response["results"])
+ return response
diff --git a/tests/unit/server/mcp/test_vector_search_timeout.py b/tests/unit/server/mcp/test_vector_search_timeout.py
index c29650b2d..151f36b96 100644
--- a/tests/unit/server/mcp/test_vector_search_timeout.py
+++ b/tests/unit/server/mcp/test_vector_search_timeout.py
@@ -98,7 +98,10 @@ def test_no_hardcoded_eight_second_vector_budget() -> None:
"""Pins the fix: every vector site reads the shared budget."""
from pathlib import Path
- root = Path(tool_search.__file__).parent
- for path in (root / "tool_search.py", *sorted((root / "_answer_pipeline").glob("*.py"))):
+ root = Path(tool_search.__file__).parent.parent
+ for path in (
+ *sorted((root / "tool_search").glob("*.py")),
+ *sorted((root / "_answer_pipeline").glob("*.py")),
+ ):
text = path.read_text(encoding="utf-8")
assert "timeout=8.0" not in text, f"{path.name} still hardcodes the old vector budget"