Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(search): name the matching symbols and lines inside each file hit (#3130)
Raghav Chamadiya6f366c2153b9
diff --git a/docs/agent/MCP_TOOLS.md b/docs/agent/MCP_TOOLS.mdindex 397cec7b1..ecb3405af 100644--- a/docs/agent/MCP_TOOLS.md+++ b/docs/agent/MCP_TOOLS.md@@ -222,6 +222,8 @@ Hybrid search that routes by the shape of the query: identifiers search the symb**Key return fields:** `results` (every row that names a file carries it in `path`, and a row naming no file has no `path`; symbol hits carry `symbol_id`, line bounds and `signature`, plus `symbols` (`name:line` of up to five other matches in that file, then `+N more`) when several matched; concept hits carry `relevance_score`, `snippet` and `sources`; `file` on symbol and file hits is a deprecated alias of `path`, removed in the next minor release, and a page keeps `target_path` only where it differs from `path`), `candidates` (up to `limit` distinct openable file paths, best first). If your next move is a Read, read `candidates`: some `results` are pages that are not files.+**Page rows** also carry `symbols`: up to three `name:line` entries (`Owner.member` for members) in that file whose names match the query's words.+```search_codebase(query="GitIndexer index_repo")search_codebase(query="login", mode="symbol", symbol_kind="method")diff --git a/packages/server/src/repowise/server/mcp_server/_hit_symbols.py b/packages/server/src/repowise/server/mcp_server/_hit_symbols.pynew file mode 100644index 000000000..b5743af98--- /dev/null+++ b/packages/server/src/repowise/server/mcp_server/_hit_symbols.py@@ -0,0 +1,126 @@+"""Name the symbols inside each file hit that the query's words point at.++A search row names a file; the caller usually wants the function in it next.+Each file-backed page row gets ``symbols``: up to :data:`_MAX_HIT_SYMBOLS`+entries of ``name:line``, the same shape a collapsed symbol row already carries+for its same-file neighbours. A member is named ``Owner.member`` so a bare+``run:40`` is never ambiguous.+"""++from __future__ import annotations++import logging++from sqlalchemy import select++from repowise.core.persistence.database import get_session+from repowise.core.persistence.models import WikiSymbol+from repowise.server.mcp_server._helpers import _get_repo+from repowise.server.mcp_server._page_paths import hit_file_path+from repowise.server.mcp_server._query_terms import content_terms+from repowise.server.mcp_server.tool_search_symbols import _tokens++_log = logging.getLogger("repowise.mcp.search")++_MAX_HIT_SYMBOLS = 3++# Values rank after the callables and types a navigation question is after.+_VALUE_KINDS = frozenset({"constant", "variable", "property"})+++def _stem(word: str) -> str:+ """``files`` -> ``file``: enough inflection folding to match prose to names."""+ return word[:-1] if len(word) > 3 and word.endswith("s") and not word.endswith("ss") else word+++def _stems(text: str | None) -> set[str]:+ return {_stem(t) for t in _tokens(text)}+++def _package_terms(terms: set[str], paths: set[str]) -> set[str]:+ """Query words in the folders of most result files: a package or repo name.++ ``flask`` in ``src/flask/*`` matches a symbol in every file under it, so it+ says nothing about which symbol is meant. A folder only one hit sits in+ (``json/``) is the topic, and keeps its word.+ """+ if len(paths) < 2:+ return set()+ counts: dict[str, int] = {}+ for path in paths:+ for term in terms & _stems(path.rpartition("/")[0]):+ counts[term] = counts.get(term, 0) + 1+ return {term for term, n in counts.items() if n * 2 > len(paths)}+++def _rank_key(row, terms: set[str]) -> tuple | None:+ """Sort key for *row*, or ``None`` when its name shares no query word."""+ name_hits = len(terms & _stems(row.name))+ if not name_hits:+ return None+ text_hits = len(terms & (_stems(row.signature) | _stems(row.docstring)))+ private = row.visibility != "public" or (row.name or "").startswith("_")+ return (-name_hits, -text_hits, row.kind in _VALUE_KINDS, private, row.start_line)+++def _label(row) -> str:+ """``Owner.member`` for a member, in the stored qualified name's separator."""+ qualified = row.qualified_name or ""+ sep = "::" if "::" in qualified else "."+ parts = qualified.rsplit(sep, 2)+ if row.parent_name and len(parts) >= 2 and parts[-1] == row.name:+ return sep.join(parts[-2:])+ return row.name+++async def attach_hit_symbols(ctx, query: str, rows: list[dict]) -> None:+ """Add ``symbols`` to each file-backed page row, in place; one query for all rows.++ Symbol rows are left alone (they already name their symbol), and a row whose+ file has no symbol sharing a query word gets no field. A failure here only+ costs the field, never the search.+ """+ try:+ await _attach(ctx, query, rows)+ except Exception:+ _log.debug("search_codebase: hit symbol lookup failed", exc_info=True)+++async def _attach(ctx, query: str, rows: list[dict]) -> None:+ terms = {_stem(t) for t in content_terms(query)}+ targets = {+ id(row): path+ for row in rows+ if row.get("type") != "symbol" and "symbols" not in row and (path := hit_file_path(row))+ }+ terms -= _package_terms(terms, set(targets.values()))+ if not terms or not targets:+ return+ async with get_session(ctx.session_factory) as session:+ repo = await _get_repo(session)+ result = await session.execute(+ select(+ WikiSymbol.file_path,+ WikiSymbol.name,+ WikiSymbol.qualified_name,+ WikiSymbol.parent_name,+ WikiSymbol.kind,+ WikiSymbol.start_line,+ WikiSymbol.signature,+ WikiSymbol.docstring,+ WikiSymbol.visibility,+ ).where(+ WikiSymbol.repository_id == repo.id,+ WikiSymbol.file_path.in_(set(targets.values())),+ WikiSymbol.start_line > 0,+ )+ )+ by_file: dict[str, list[tuple]] = {}+ for row in result.all():+ key = _rank_key(row, terms)+ if key is not None:+ by_file.setdefault(row.file_path, []).append((key, _label(row), row.start_line))+ for hit in rows:+ ranked = sorted(by_file.get(targets.get(id(hit), ""), ()), key=lambda r: r[0])+ if ranked:+ hit["symbols"] = [f"{name}:{line}" for _, name, line in ranked[:_MAX_HIT_SYMBOLS]]diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pyindex b52320f55..9afb3a607 100644--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py@@ -37,6 +37,7 @@ from repowise.server.mcp_server._helpers import (resolve_enum_argument,vector_search_timeout_s,)+from repowise.server.mcp_server._hit_symbols import attach_hit_symbolsfrom repowise.server.mcp_server._meta import EXHAUSTIVE_SWEEP_HINTfrom repowise.server.mcp_server._meta import build_meta as _build_metafrom repowise.server.mcp_server._page_paths import add_row_paths, file_candidates, hit_file_path@@ -1015,6 +1016,7 @@ async def _structured_search(c = await _search_single_repo(ctx, query, limit, page_type, kind)for item in c:item["type"] = "page"+ await attach_hit_symbols(ctx, query, c)_tag_repo(c, ctx, multi)concepts.extend(c)@@ -1175,7 +1177,8 @@ async def search_codebase(then file-backed pages only). Decision records rank below file pagesunless the query is why-shaped.- Rows naming a file carry `path`; `candidates` lists up to `limit`+ Rows naming a file carry `path`; concept pages add `symbols`+ (name:line) the query matches. `candidates` lists up to `limit`distinct files to Read, best first.Args:@@ -1312,6 +1315,7 @@ async def search_codebase(# below can reach past it. See its comment for why that matters.ranked = list(output)output = output[:limit]+ await attach_hit_symbols(ctx, query, output)# Derive confidence_score from relative position in the result set._assign_confidence(output, "relevance_score", "confidence_score")diff --git a/tests/unit/server/mcp/test_search.py b/tests/unit/server/mcp/test_search.pyindex 1262e3e5b..52f0b8edd 100644--- a/tests/unit/server/mcp/test_search.py+++ b/tests/unit/server/mcp/test_search.py@@ -2014,3 +2014,169 @@ class TestSymbolModeExactOnly:assert res["results"]assert "fuzzy_omitted" not in resassert "exactly matches" in res["note"]+++class TestHitSymbols:+ """File rows name the symbols inside them that the query's words match."""++ @staticmethod+ async def _seed_symbols(session, rid):+ from repowise.core.persistence.models import WikiSymbol++ wav = "services/wav_metadata.py"+ rows = [+ (wav, "write_bext_chunk", 42, "public", "def write_bext_chunk(f)", "function", None),+ (wav, "_list_info_bytes", 88, "private", "def _list_info_bytes()", "function", None),+ (wav, "read_header", 10, "public", "def read_header(f)", "function", None),+ (wav, "bext_version", 3, "public", "bext_version = 2", "constant", None),+ (wav, "stamp_list_info", 120, "public", "def stamp_list_info(self)", "method", "WavWriter"),+ # No real line: never served.+ (wav, "bext_info_list", 0, "public", "def bext_info_list()", "function", None),+ ("services/mixer.py", "mix_tracks", 5, "public", "def mix_tracks()", "function", None),+ ]+ for i, (path, name, line, vis, sig, kind, parent) in enumerate(rows):+ qualified = f"services.wav_metadata.{parent}.{name}" if parent else name+ session.add(+ WikiSymbol(+ id=f"hit-sym-{i}",+ repository_id=rid,+ file_path=path,+ symbol_id=f"{path}::{name}",+ name=name,+ qualified_name=qualified,+ parent_name=parent,+ kind=kind,+ signature=sig,+ start_line=line,+ end_line=line + 5,+ visibility=vis,+ language="python",+ )+ )+ await session.commit()++ @staticmethod+ def _fake_vector():+ import repowise.server.mcp_server as mcp_mod++ async def fake_search(query, limit=10):+ return [+ _mk_result(+ "file_page:services/wav_metadata.py",+ "WAV metadata",+ "file_page",+ "services/wav_metadata.py",+ 0.8,+ ),+ _mk_result(+ "file_page:services/mixer.py", "Mixer", "file_page", "services/mixer.py", 0.6+ ),+ ]++ mcp_mod._vector_store.search = fake_search++ @pytest.mark.asyncio+ async def test_a_prose_query_names_the_matching_symbol_and_line(+ self, session, populated_db, setup_mcp+ ):+ from repowise.server.mcp_server import search_codebase++ await self._seed_symbols(session, populated_db)+ await _seed_page("file_page:services/wav_metadata.py", "services/wav_metadata.py")+ await _seed_page("file_page:services/mixer.py", "services/mixer.py")+ self._fake_vector()++ res = await search_codebase("which function stamps bext and LIST info into wav files")+ by_path = {r["path"]: r for r in res["results"]}+ # Most query words first; a function outranks an equally matched constant.+ assert by_path["services/wav_metadata.py"]["symbols"] == [+ "WavWriter.stamp_list_info:120",+ "_list_info_bytes:88",+ "write_bext_chunk:42",+ ]+ # No symbol in the mixer shares a word with the query.+ assert "symbols" not in by_path["services/mixer.py"]++ hybrid = await search_codebase(+ "where does wav_metadata stamp bext or mix tracks", mode="hybrid"+ )+ by_path = {r["path"]: r for r in hybrid["results"]}+ # The hybrid page leg names symbols; the symbol row standing in for+ # wav_metadata.py already names its own and gets no list.+ assert by_path["services/mixer.py"]["symbols"] == ["mix_tracks:5"]+ assert by_path["services/wav_metadata.py"]["type"] == "symbol"+ assert "symbols" not in by_path["services/wav_metadata.py"]++ @pytest.mark.asyncio+ async def test_one_symbol_query_serves_every_file_row(+ self, engine, session, populated_db, setup_mcp+ ):+ from sqlalchemy import event++ from repowise.server.mcp_server import search_codebase++ await self._seed_symbols(session, populated_db)+ await _seed_page("file_page:services/wav_metadata.py", "services/wav_metadata.py")+ await _seed_page("file_page:services/mixer.py", "services/mixer.py")+ self._fake_vector()++ statements: list[str] = []++ def record(conn, cursor, statement, *args):+ statements.append(statement)++ event.listen(engine.sync_engine, "before_cursor_execute", record)+ try:+ res = await search_codebase("which function stamps bext and mix tracks", mode="concept")+ finally:+ event.remove(engine.sync_engine, "before_cursor_execute", record)++ assert {r["path"] for r in res["results"] if r.get("symbols")} == {+ "services/wav_metadata.py",+ "services/mixer.py",+ }+ file_scoped = [s for s in statements if "wiki_symbols.file_path IN" in s]+ assert len(file_scoped) == 1++ def test_a_package_word_shared_by_most_hits_names_no_symbol(self):+ from repowise.server.mcp_server._hit_symbols import _package_terms++ paths = {"src/flask/app.py", "src/flask/cli.py", "src/flask/json/provider.py"}+ # "flask" is every hit's package; "json" is one hit's topic.+ assert _package_terms({"flask", "json", "route"}, paths) == {"flask"}+ assert _package_terms({"flask"}, {"src/flask/app.py"}) == set()++ @pytest.mark.asyncio+ async def test_a_lone_symbol_row_gets_no_list(self, session, populated_db, setup_mcp):+ from repowise.server.mcp_server._helpers import _resolve_repo_context+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols++ await self._seed_symbols(session, populated_db)+ symbol_row = {"type": "symbol", "name": "write_bext_chunk", "file": "services/wav_metadata.py"}+ page_row = {"page_type": "file_page", "target_path": "services/wav_metadata.py"}+ ctx = await _resolve_repo_context(None)+ await attach_hit_symbols(ctx, "stamp bext", [symbol_row, page_row])+ assert "symbols" not in symbol_row+ assert page_row["symbols"][0] == "write_bext_chunk:42"++ @pytest.mark.asyncio+ async def test_a_failed_lookup_leaves_the_response_intact(+ self, session, populated_db, setup_mcp, monkeypatch+ ):+ from repowise.server.mcp_server import _hit_symbols, search_codebase++ await self._seed_symbols(session, populated_db)+ await _seed_page("file_page:services/wav_metadata.py", "services/wav_metadata.py")+ await _seed_page("file_page:services/mixer.py", "services/mixer.py")+ self._fake_vector()++ async def boom(*args, **kwargs):+ raise RuntimeError("symbol table unavailable")++ monkeypatch.setattr(_hit_symbols, "_attach", boom)+ res = await search_codebase("which function stamps bext into wav files", mode="concept")+ assert [r["path"] for r in res["results"]] == [+ "services/wav_metadata.py",+ "services/mixer.py",+ ]+ assert not any("symbols" in r for r in res["results"])