Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(search): surface symbols whose name words match a prose query (#3194)
Raghav Chamadiyadc5fd220a3c1
diff --git a/packages/server/src/repowise/server/mcp_server/_prose_symbols.py b/packages/server/src/repowise/server/mcp_server/_prose_symbols.pyindex 55fa4ea1f..9891fe02a 100644--- a/packages/server/src/repowise/server/mcp_server/_prose_symbols.py+++ b/packages/server/src/repowise/server/mcp_server/_prose_symbols.py@@ -40,20 +40,23 @@ prose-to-symbol search useless.from __future__ import annotations-from typing import Any+from collections import Counter+from collections.abc import Iterable+from typing import Any, NamedTuple-from sqlalchemy import func, or_, select+from sqlalchemy import selectfrom repowise.core.persistence.database import get_sessionfrom repowise.core.persistence.models import GraphNode, Page, WikiSymbolfrom repowise.core.test_paths import is_test_path, is_test_related_path+from repowise.server.mcp_server._graph_files import per_indexfrom repowise.server.mcp_server._helpers import (- LIKE_ESCAPE,_get_exclude_spec,_get_repo,- escape_like,is_excluded,)+from repowise.server.mcp_server._hit_symbols import _stem+from repowise.server.mcp_server._query_terms import STOPWORDSfrom repowise.server.mcp_server._query_terms import content_terms as content_termsfrom repowise.server.mcp_server.tool_search_symbols import (_symbol_kind_for_request_kind,@@ -86,31 +89,106 @@ _LEAF_NAME_BONUS = 15.0# the question; the bare member name alone is deliberately insufficient._GENERIC_MEMBER_NAMES = frozenset({"get", "run", "main"})+# A name built out of the question's words: at least this many of them, making+# up at least this share of the name's own words. Two common words meeting in+# one short name is rare even when each word alone is everywhere.+_NAME_COVER_MIN_WORDS = 2+_NAME_COVER_MIN_SHARE = 2 / 3-async def _candidates_for_term(session, repo_id: str, term: str) -> tuple[list[WikiSymbol], bool]:++class _SymbolTextIndex(NamedTuple):+ """One repo's symbols as plain text, by position, for in-process matching."""++ symbol_ids: tuple[str, ...]+ names: tuple[str, ...]+ # Lowercased name, qualified name and path, space-joined (no term has a+ # space, so none can match across two fields).+ haystacks: tuple[str, ...]+ postings: dict[str, tuple[int, ...]] # name word -> positions whose name has it+++async def _symbol_text_index(session, repo_id: str) -> _SymbolTextIndex:+ """Built once per index state; every question after that matches in memory.++ Scanning the symbol table per term with ``LIKE`` cost hundreds of+ milliseconds a term on a large repo, which made this leg the slowest part+ of a search. The table only changes with the index.+ """++ async def build() -> _SymbolTextIndex:+ res = await session.execute(+ select(+ WikiSymbol.symbol_id,+ WikiSymbol.name,+ WikiSymbol.qualified_name,+ WikiSymbol.file_path,+ ).where(WikiSymbol.repository_id == repo_id)+ )+ rows = res.all()+ postings: dict[str, list[int]] = {}+ for i, row in enumerate(rows):+ for word in _name_words(row[1]):+ postings.setdefault(word, []).append(i)+ return _SymbolTextIndex(+ tuple(row[0] for row in rows),+ tuple(row[1] or "" for row in rows),+ tuple(f"{row[1] or ''} {row[2] or ''} {row[3] or ''}".lower() for row in rows),+ {word: tuple(pos) for word, pos in postings.items()},+ )++ return await per_index(session, repo_id, "prose_symbol_text", build)+++def _candidates_for_term(index: _SymbolTextIndex, term: str) -> tuple[list[int], bool]:"""Up to :data:`_PER_TERM_CANDIDATES` symbols whose identity mentions *term*.- Returns ``(rows, saturated)``. Shorter names first: a term is a larger+ Returns ``(positions, saturated)``. Shorter names first: a term is a largershare of a short name, so ``persist`` identifies ``persist_pages`` farmore strongly than it identifies ``_persist_symbols_for_changed_files``,and the window should not be spent on the latter."""- esc = escape_like(term)- stmt = (- select(WikiSymbol)- .where(- WikiSymbol.repository_id == repo_id,- or_(- WikiSymbol.name.ilike(f"%{esc}%", escape=LIKE_ESCAPE),- WikiSymbol.qualified_name.ilike(f"%{esc}%", escape=LIKE_ESCAPE),- WikiSymbol.file_path.ilike(f"%{esc}%", escape=LIKE_ESCAPE),- ),- )- .order_by(func.length(WikiSymbol.name))- .limit(_PER_TERM_CANDIDATES)+ needle = term.lower()+ hits = [i for i, text in enumerate(index.haystacks) if needle in text]+ hits.sort(key=lambda i: len(index.names[i]))+ window = hits[:_PER_TERM_CANDIDATES]+ return window, len(window) >= _PER_TERM_CANDIDATES+++def _name_cover_candidates(index: _SymbolTextIndex, terms: list[str]) -> list[int]:+ """Symbols whose name the query's words mostly spell (see :func:`_name_covered`).++ The per-term windows rank by name length, so a long name that combines two+ saturated words (``dynamic`` + ``import``) falls outside both of them. This+ asks for the combination directly, from the inverted name words.+ """+ stems = {_stem(t) for t in terms}+ if len(stems) < _NAME_COVER_MIN_WORDS:+ return []+ hits: Counter[int] = Counter()+ for stem in stems:+ hits.update(index.postings.get(stem, ()))+ covered = [+ i+ for i, n in hits.items()+ if n >= _NAME_COVER_MIN_WORDS+ and n >= _NAME_COVER_MIN_SHARE * len(_name_words(index.names[i]))+ ]+ covered.sort(key=lambda i: (-hits[i], len(index.names[i]), index.symbol_ids[i]))+ return covered[:_PER_TERM_CANDIDATES]+++def _name_words(name: str | None) -> set[str]:+ """The stemmed non-stopword words of a symbol's own name."""+ return {_stem(t) for t in _tokens(name) if t not in STOPWORDS}+++def _name_covered(row: WikiSymbol, terms: Iterable[str]) -> bool:+ """Whether the question's words make up most of *row*'s name."""+ words = _name_words(row.name)+ hit = words & {_stem(t) for t in terms}+ return (+ len(hit) >= _NAME_COVER_MIN_WORDS and len(hit) >= _NAME_COVER_MIN_SHARE * len(words))- rows = list((await session.execute(stmt)).scalars().all())- return rows, len(rows) >= _PER_TERM_CANDIDATESdef _score(@@ -158,6 +236,8 @@ def _corroborated(row: WikiSymbol, covered: dict[str, float], saturated: set[strreturn Falsename = (row.name or "").lower()informative = {term for term in covered if term not in saturated}+ if _name_covered(row, set(covered)):+ return Trueif name in _GENERIC_MEMBER_NAMES:return any(term != name for term in informative)if name in informative:@@ -187,18 +267,26 @@ async def search_symbols_by_terms(async with get_session(ctx.session_factory) as session:repository = await _get_repo(session)- by_id: dict[str, WikiSymbol] = {}+ index = await _symbol_text_index(session, repository.id)matched: dict[str, set[str]] = {}saturated: set[str] = set()for term in terms:- rows, is_saturated = await _candidates_for_term(session, repository.id, term)+ window, is_saturated = _candidates_for_term(index, term)if is_saturated:saturated.add(term)- for row in rows:- by_id[row.symbol_id] = row- matched.setdefault(row.symbol_id, set()).add(term)- if not by_id:+ for i in window:+ matched.setdefault(index.symbol_ids[i], set()).add(term)+ for i in _name_cover_candidates(index, terms):+ matched.setdefault(index.symbol_ids[i], set())+ if not matched:return []+ res = await session.execute(+ select(WikiSymbol).where(+ WikiSymbol.repository_id == repository.id,+ WikiSymbol.symbol_id.in_(list(matched)),+ )+ )+ by_id = {row.symbol_id: row for row in res.scalars().all()}gres = await session.execute(select(GraphNode).where(@@ -232,6 +320,11 @@ async def search_symbols_by_terms(for t in matched.get(symbol_id, ())if _covers(t, stoks)}+ # Words that together spell most of the name are one signal, not+ # several common words, so they count in full.+ if _name_covered(row, terms):+ words = _name_words(row.name)+ covered.update({t: 1.0 for t in terms if _stem(t) in words})if not _corroborated(row, covered, saturated):continuescored.append((_score(row, gnode, covered), row))diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pyindex 0a770b547..5eb624976 100644--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py@@ -21,7 +21,12 @@ from repowise.core.providers.embedding import store_has_semantic_vectorsfrom repowise.core.registry import ToolRecipefrom repowise.core.registry import mcp_tool_registry as mcpfrom repowise.core.test_paths import is_test_path, is_test_related_path-from repowise.server.mcp_server._answer_pipeline import _RRF_K, _RRF_SCORE_SCALE+from repowise.server.mcp_server._answer_pipeline import (+ _RRF_K,+ _RRF_SCORE_SCALE,+ _SYMBOL_LEG_RRF_K,+ _safe_symbol_search,+)from repowise.server.mcp_server._budget import (OmissionCollector,register_post_enforce,@@ -42,8 +47,12 @@ from repowise.server.mcp_server._hit_symbols import attach_hit_symbolsfrom repowise.server.mcp_server._line_hits import attach_line_hitsfrom repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINTfrom repowise.server.mcp_server._meta import build_meta as _build_meta-from repowise.server.mcp_server._page_paths import add_row_paths, file_candidates, hit_file_path-from repowise.server.mcp_server._prose_symbols import symbol_backed_pages+from repowise.server.mcp_server._page_paths import (+ add_row_paths,+ file_candidates,+ file_path_of,+ hit_file_path,+)from repowise.server.mcp_server._query_shape import (_DECISION_DOWNWEIGHT,_MIN_RELEVANCE_SCORE,@@ -408,47 +417,23 @@ def _symbol_tail_reserve(limit: int) -> int:return limit // _SYMBOL_TAIL_DIVISOR-async def _append_symbol_backed(- ctx, query: str, output: list[dict], limit: int, kind: str | None-) -> list[dict]:- """Give the weakest tail slots to files the symbol index reached.+def _reserve_symbol_tail(output: list[dict], limit: int, kind: str | None) -> list[dict]:+ """Give the weakest window slots to symbol-leg files ranked past ``limit``.- Concept hits keep the head. Symbol-backed files not already in the window- take up to :func:`_symbol_tail_reserve` slots, and the concept hits they- displaced fall in behind them (nothing is dropped before the caller's- ``limit`` cut). Returns ``output`` unchanged when the symbol leg is empty- or every file it named is already present.+ Concept hits keep the head. Files the symbol leg reached that fusion left+ outside the window take up to :func:`_symbol_tail_reserve` slots, and the+ concept hits they displaced fall in behind them (nothing is dropped)."""reserve = _symbol_tail_reserve(limit)if reserve <= 0 or kind in ("config", "doc"):# config/doc windows are asking for pages with no symbols in them.return output- pages = await symbol_backed_pages(ctx, query, max_files=reserve * 2)- if not pages:- return output-- present = {item.get("target_path") for item in output}- extra = [p for p in pages if p["target_path"] not in present][:reserve]+ extra = [item for item in output[limit:] if "symbol" in item.get("sources", ())][:reserve]if not extra:return output- # Score them just under the last concept hit so the ordering the caller- # sees stays monotone; they are additions to the window, not a re-ranking- # of it.- floor = min((item.get("relevance_score") or 0.0) for item in output) if output else 0.0- additions = [- {- "page_id": p["page_id"],- "title": p["title"],- "page_type": p["page_type"],- "target_path": p["target_path"],- "snippet": (p.get("summary") or "")[:200],- "relevance_score": round(max(floor - 0.001 * (i + 1), 0.0), 4),- "sources": ["symbol"],- }- for i, p in enumerate(extra)- ]- head = output[: max(0, limit - len(additions))]- return head + additions + output[len(head) :]+ head = output[: limit - len(extra)]+ taken = {id(item) for item in head + extra}+ return head + extra + [item for item in output if id(item) not in taken]# Path-prefix heuristics for the ``kind`` filter. We classify a hit's@@ -702,6 +687,23 @@ async def _safe_vector(ctx, query: str, limit: int) -> list:return []+def _symbol_leg_target(fused: dict[str, dict], page_id: str) -> str:+ """The fused page a symbol-leg file page credits: the best page of its file.++ The leg names files, while the page legs may have reached the same file+ through a symbol page. Crediting only the file page id would leave that+ hit unlifted and add a weak twin that collapsing then drops.+ """+ path = page_id.partition(":")[2]+ best = None+ for pid, entry in fused.items():+ if file_path_of(entry["page_type"], pid.partition(":")[2]) != path:+ continue+ if best is None or entry["_rrf"] > fused[best]["_rrf"]:+ best = pid+ return best or page_id++def _fused_entry(r) -> dict:"""Seed a fused-result dict from a retriever hit (RRF score added by caller)."""return {@@ -715,13 +717,13 @@ def _fused_entry(r) -> dict:async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | None) -> list[dict]:- """Retrieve by fusing FTS and vector search via Reciprocal Rank Fusion.+ """Retrieve by fusing FTS, vector and symbol search via Reciprocal Rank Fusion.- Both retrievers run in parallel; a hit's score is the sum of+ The retrievers run in parallel; a hit's score is the sum of``1/(rank + k)`` over the retrievers that surfaced it, scaled into theBM25 range the downstream relevance gates were tuned against. Each hit- carries a ``sources`` list (``"fts"``, ``"vector"``, or both) — a page- found by both retrievers is a stronger match than one found by one mode.+ carries a ``sources`` list (``"fts"``, ``"vector"``, ``"symbol"``): a page+ found by several retrievers is a stronger match than one found by one mode.This mirrors get_answer's ``hybrid_retrieve`` so the two entry points rankthe same corpus identically. The prior path ran vector search and fell@@ -735,9 +737,10 @@ async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | No(for RRF) is the hit's place in the retriever's own list, unchanged by thefloor."""- fts_results, vec_results = await asyncio.gather(+ fts_results, vec_results, sym_results = await asyncio.gather(_safe_fts(ctx, query, fetch_limit),_safe_vector(ctx, query, fetch_limit),+ _safe_symbol_search(ctx, query),)fused: dict[str, dict] = {}@@ -753,6 +756,12 @@ async def _fused_retrieve(ctx, query: str, fetch_limit: int, page_type: str | Noentry = fused.setdefault(r.page_id, _fused_entry(r))entry["_rrf"] += 1.0 / (rank + _RRF_K)entry["_sources"].add("fts")+ # File pages whose symbol names the query's words spell. Weighted below the+ # page legs, as in get_answer; it has no raw score to floor.+ for rank, r in enumerate(sym_results):+ entry = fused.setdefault(_symbol_leg_target(fused, r.page_id), _fused_entry(r))+ entry["_rrf"] += 1.0 / (rank + _SYMBOL_LEG_RRF_K)+ entry["_sources"].add("symbol")output: list[dict] = []for entry in fused.values():@@ -1323,8 +1332,7 @@ async def search_codebase(# Files the page retrievers structurally cannot see (a private helper, a# local name, anything a file page's public-symbol table omits) get the# weakest tail slots. No-op when the symbol leg names nothing new.- with contextlib.suppress(Exception):- output = await _append_symbol_backed(ctx, query, output, limit, kind)+ output = _reserve_symbol_tail(output, limit, kind)_drop_internal_ranking_fields(output)# The full ranked pool, kept before the caller's cut so ``candidates``# below can reach past it. See its comment for why that matters.diff --git a/tests/unit/server/mcp/test_prose_symbol_names.py b/tests/unit/server/mcp/test_prose_symbol_names.pynew file mode 100644index 000000000..9db84755e--- /dev/null+++ b/tests/unit/server/mcp/test_prose_symbol_names.py@@ -0,0 +1,191 @@+"""A prose query whose words spell a symbol's name reaches that symbol and its file.++Synthetic symbols in Python, TypeScript, Go and C#, plus distractors that share+one common word with the queries.+"""++from __future__ import annotations++import json+from datetime import UTC, datetime+from types import SimpleNamespace++import pytest++from repowise.core.persistence.models import Page, WikiSymbol+from repowise.server.mcp_server import _prose_symbols as prose_symbols+from repowise.server.mcp_server._prose_symbols import (+ _name_covered,+ content_terms,+ search_symbols_by_terms,+ symbol_backed_pages,+)++_NOW = datetime(2026, 3, 19, 12, 0, 0, tzinfo=UTC)++_SYMBOLS = [+ ("src/ingest/parser.py", "_is_dynamic_esm_import", "python"),+ ("web/src/loader.ts", "resolveLazyModulePath", "typescript"),+ ("internal/queue/retry.go", "ScheduleRetryBackoff", "go"),+ ("Services/OrderEvents.cs", "PublishOrderShippedEvent", "csharp"),+ # Distractors: each shares exactly one word with some query below.+ ("src/http/errors.py", "ErrorHandler", "python"),+ ("web/src/upload.ts", "UploadHandler", "typescript"),+ ("internal/queue/worker.go", "RetryCount", "go"),+ ("src/ingest/config.py", "dynamic_config", "python"),+ ("src/model/value.py", "is_the_value", "python"),+ ("src/ingest/esm.py", "esm", "python"),+ ("web/src/lazy.ts", "lazy", "typescript"),+ ("internal/queue/backoff.go", "Backoff", "go"),+ ("Services/Order.cs", "Order", "csharp"),+]+++async def _seed(session, rid: str) -> None:+ for i, (path, name, language) in enumerate(_SYMBOLS):+ session.add(+ WikiSymbol(+ id=f"pss{i}",+ repository_id=rid,+ file_path=path,+ symbol_id=f"{path}::{name}",+ name=name,+ qualified_name=name,+ kind="function",+ signature=name,+ start_line=10,+ end_line=20,+ docstring="",+ visibility="public",+ is_async=False,+ complexity_estimate=1,+ language=language,+ parent_name=None,+ created_at=_NOW,+ updated_at=_NOW,+ )+ )+ session.add(+ Page(+ id=f"file_page:{path}",+ repository_id=rid,+ page_type="file_page",+ title=f"File: {path}",+ content="overview",+ target_path=path,+ source_hash=f"h{i}",+ model_name="mock",+ provider_name="mock",+ generation_level=2,+ confidence=0.5,+ freshness_status="fresh",+ metadata_json="{}",+ created_at=_NOW,+ updated_at=_NOW,+ )+ )+ await session.commit()+++async def ctx(session, repo_id, factory, tmp_path, monkeypatch):+ # A one-row window per term stands in for a large repo, where every word in+ # these queries fills its window with shorter names.+ monkeypatch.setattr(prose_symbols, "_PER_TERM_CANDIDATES", 1)+ await _seed(session, repo_id)+ return SimpleNamespace(session_factory=factory, path=str(tmp_path))+++async def _names(ctx, question: str) -> list[str]:+ hits = await search_symbols_by_terms(ctx, content_terms(question), 10)+ return [h["name"] for h in hits]+++ ("question", "name", "path"),+ [+ ("how are dynamic esm imports detected", "_is_dynamic_esm_import", "src/ingest/parser.py"),+ ("resolve the lazy module path", "resolveLazyModulePath", "web/src/loader.ts"),+ ("retry backoff scheduling", "ScheduleRetryBackoff", "internal/queue/retry.go"),+ ("publish an event when an order shipped", "PublishOrderShippedEvent", "Services/OrderEvents.cs"),+ ],+)+async def test_prose_spelling_a_name_reaches_the_symbol_and_its_file(ctx, question, name, path):+ assert (await _names(ctx, question))[0] == name+ pages = await symbol_backed_pages(ctx, question, max_files=5)+ assert pages[0]["target_path"] == path+++async def test_one_shared_common_word_pulls_nothing(ctx):+ assert await _names(ctx, "which handler logs failures") == []+++async def test_one_word_of_a_longer_name_does_not_cover_it(ctx):+ # "retry" alone is one of RetryCount's two words, not two.+ assert "RetryCount" not in await _names(ctx, "retry failed jobs later")+++async def test_stopwords_are_not_name_words(ctx):+ # ``is_the_value`` has one real word; "is" and "the" cannot make up a second.+ assert "is_the_value" not in await _names(ctx, "is the value set")+++class _Sym:+ def __init__(self, name):+ self.name = name+++def test_name_cover_share_ignores_stopword_parts():+ # dynamic + import are two of the three real words (is is a stopword).+ assert _name_covered(_Sym("_is_dynamic_esm_import"), ["dynamic", "import"])+ assert not _name_covered(_Sym("_is_dynamic_esm_import_cache_key"), ["dynamic", "import"])+ assert not _name_covered(_Sym("RequestHandler"), ["handler"])+++async def test_identifier_query_is_unchanged_by_the_symbol_leg(setup_mcp, monkeypatch):+ from repowise.server.mcp_server import search_codebase, tool_search++ with_leg = await search_codebase("AuthService")++ async def no_leg(ctx, question):+ return []++ monkeypatch.setattr(tool_search, "_safe_symbol_search", no_leg)+ without_leg = await search_codebase("AuthService")+ with_leg["_meta"].pop("timing", None)+ without_leg["_meta"].pop("timing", None)+ assert json.dumps(with_leg, sort_keys=True, default=str) == json.dumps(+ without_leg, sort_keys=True, default=str+ )+++async def test_concept_search_lifts_the_file_a_prose_query_names(setup_mcp, monkeypatch):+ import repowise.server.mcp_server as mcp_mod+ from repowise.core.persistence.search import SearchResult+ from repowise.server.mcp_server import search_codebase++ async def fake_search(query, limit=10):+ # The page retrievers prefer middleware.py; only the symbol name says service.py.+ return [+ SearchResult(+ page_id=f"file_page:{path}",+ title=path,+ page_type="file_page",+ target_path=path,+ score=score,+ snippet=path,+ search_type="vector",+ )+ for path, score in (("src/auth/middleware.py", 0.62), ("src/auth/service.py", 0.61))+ ]++ mcp_mod._vector_store.search = fake_search+ result = await search_codebase("what does the auth service class do", mode="concept")+ assert result["results"][0]["path"] == "src/auth/service.py"+ assert "symbol" in result["results"][0]["sources"]diff --git a/tests/unit/server/mcp/test_search.py b/tests/unit/server/mcp/test_search.pyindex a6ca927bc..adadc7807 100644--- a/tests/unit/server/mcp/test_search.py+++ b/tests/unit/server/mcp/test_search.py@@ -547,7 +547,9 @@ class TestNoiseDemotion:]mcp_mod._vector_store.search = fake_search- result = await search_codebase("how is the auth service tested")+ # Not "auth service": that spells ``AuthService``, whose file the+ # symbol leg would rightly lift on its own.+ result = await search_codebase("how is the auth flow tested")assert result["results"][0]["path"] == "tests/unit/test_service.py"