Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(search): return the best-matching lines for prose queries (#3189)
Raghav Chamadiyae53e753370bf
diff --git a/docs/agent/MCP_TOOLS.md b/docs/agent/MCP_TOOLS.mdindex 2680d51ba..5526c5c4b 100644--- a/docs/agent/MCP_TOOLS.md+++ b/docs/agent/MCP_TOOLS.md@@ -223,7 +223,7 @@ Hybrid search that routes by the shape of the query: identifiers search the symb**Key return fields:** `results` (every row that names a file carries it in `path`, and a row naming no file has no `path`; symbol hits carry `symbol_id`, line bounds and `signature`, plus `symbols` (`name:line` of up to five other matches in that file, then `+N more`) when several matched; concept hits carry `relevance_score`, `snippet` and `sources`; `file` on symbol and file hits is a deprecated alias of `path`, removed in the next minor release, and a page keeps `target_path` only where it differs from `path`), `candidates` (up to `limit` distinct openable file paths, best first). If your next move is a Read, read `candidates`: some `results` are pages that are not files.-**Page rows** also carry `symbols`: up to three `name:line` entries (`Owner.member` for members) in that file whose names match the query's words.+**Page rows** also carry `symbols`: up to three `name:line` entries (`Owner.member` for members) in that file whose names match the query's words. The first three page rows whose file has a matching line also carry `matched_lines`: a sample of up to three `{line, text}` source lines read from the live file that share the most query words, code lines before comments and imports. Unlike `lines`, it is never every match.An identifier or literal query also returns `lines` (`{path, line, kind, text}`, definitions first, at most 50): the `get_context` `references` of one to three exact symbols, else a live scan for the token (kind `definition` or `match`). `complete` is true only when they are every match (see `references` above); otherwise `reasons` says why.diff --git a/packages/server/src/repowise/server/mcp_server/_edit_sites.py b/packages/server/src/repowise/server/mcp_server/_edit_sites.pyindex c54dfcb1e..d082cde41 100644--- a/packages/server/src/repowise/server/mcp_server/_edit_sites.py+++ b/packages/server/src/repowise/server/mcp_server/_edit_sites.py@@ -84,7 +84,11 @@ def _read_lines(root: Path, rel: str) -> list[str] | None:except OSError:return Nonetext = read_repo_file_text(root, rel)- return None if text is None else text.splitlines()+ if text is None:+ return None+ # Only a newline ends a line: splitlines() also splits on form feeds and+ # other separators, which would shift every line number after them.+ return [line.removesuffix("\r") for line in text.removesuffix("\n").split("\n")]def _scan(diff --git a/packages/server/src/repowise/server/mcp_server/_hit_symbols.py b/packages/server/src/repowise/server/mcp_server/_hit_symbols.pyindex b5743af98..e408e3b83 100644--- a/packages/server/src/repowise/server/mcp_server/_hit_symbols.py+++ b/packages/server/src/repowise/server/mcp_server/_hit_symbols.py@@ -4,17 +4,23 @@ A search row names a file; the caller usually wants the function in it next.Each file-backed page row gets ``symbols``: up to :data:`_MAX_HIT_SYMBOLS`entries of ``name:line``, the same shape a collapsed symbol row already carriesfor its same-file neighbours. A member is named ``Owner.member`` so a bare-``run:40`` is never ambiguous.+``run:40`` is never ambiguous. The first :data:`_MAX_LINE_FILES` rows whose live+file has a line sharing query words also get ``matched_lines``: a sample of the+best such lines, not every match."""from __future__ import annotations+import asyncioimport logging+import re+from pathlib import Pathfrom sqlalchemy import selectfrom repowise.core.persistence.database import get_sessionfrom repowise.core.persistence.models import WikiSymbol+from repowise.server.mcp_server._edit_sites import MAX_TEXT_CHARS, _import_lines, _read_linesfrom repowise.server.mcp_server._helpers import _get_repofrom repowise.server.mcp_server._page_paths import hit_file_pathfrom repowise.server.mcp_server._query_terms import content_terms@@ -23,6 +29,12 @@ from repowise.server.mcp_server.tool_search_symbols import _tokens_log = logging.getLogger("repowise.mcp.search")_MAX_HIT_SYMBOLS = 3+# Only the top files are read: past them an agent opens the file anyway.+_MAX_LINE_FILES = 3+_MAX_HIT_LINES = 3+# Files tried for lines, so unreadable or unmatched ones cannot make it unbounded.+_MAX_LINE_READS = 2 * _MAX_LINE_FILES+_COMMENT_ONLY = re.compile(r"^\s*(?:#|//|/\*|\*|--|<!--)")# Values rank after the callables and types a navigation question is after._VALUE_KINDS = frozenset({"constant", "variable", "property"})@@ -73,12 +85,51 @@ def _label(row) -> str:return row.name+def best_lines(+ lines: list[str], terms: set[str], spans: list[tuple[int, int]]+) -> list[dict]:+ """``{line, text}`` for the lines sharing the most of *terms*, best first.++ A line inside a symbol whose name matches wins a tie; comment-only and+ import lines are served only when no other line matches. With two or more+ terms a line must match two, so one common word does not decide alone.+ """+ need = min(2, len(terms))+ imports = _import_lines(lines)+ scored = []+ for n, text in enumerate(lines, 1):+ hits = len(terms & _stems(text))+ if hits < need:+ continue+ weak = n in imports or bool(_COMMENT_ONLY.match(text))+ inside = any(start <= n <= end for start, end in spans)+ scored.append((weak, -hits, not inside, n, text))+ if any(not s[0] for s in scored):+ scored = [s for s in scored if not s[0]]+ best = sorted(scored)[:_MAX_HIT_LINES]+ return [{"line": n, "text": text.strip()[:MAX_TEXT_CHARS]} for *_, n, text in best]+++def _sample_lines(+ root: Path, wanted: list[tuple[str, list[tuple[int, int]]]], terms: set[str]+) -> dict[str, list[dict]]:+ """Best lines for the first files in *wanted* that yield any, read in order."""+ found: dict[str, list[dict]] = {}+ for path, spans in wanted[:_MAX_LINE_READS]:+ lines = _read_lines(root, path)+ if lines and (best := best_lines(lines, terms, spans)):+ found[path] = best+ if len(found) >= _MAX_LINE_FILES:+ break+ return found++async def attach_hit_symbols(ctx, query: str, rows: list[dict]) -> None:- """Add ``symbols`` to each file-backed page row, in place; one query for all rows.+ """Add ``symbols`` (and ``matched_lines`` to top rows) to file-backed page rows, in place.- Symbol rows are left alone (they already name their symbol), and a row whose- file has no symbol sharing a query word gets no field. A failure here only- costs the field, never the search.+ One symbol query serves all rows. Symbol rows are left alone (they already+ name their symbol), and a row with nothing sharing a query word gets no+ field. A failure here only costs the fields, never the search."""try:await _attach(ctx, query, rows)@@ -106,6 +157,7 @@ async def _attach(ctx, query: str, rows: list[dict]) -> None:WikiSymbol.parent_name,WikiSymbol.kind,WikiSymbol.start_line,+ WikiSymbol.end_line,WikiSymbol.signature,WikiSymbol.docstring,WikiSymbol.visibility,@@ -119,8 +171,20 @@ async def _attach(ctx, query: str, rows: list[dict]) -> None:for row in result.all():key = _rank_key(row, terms)if key is not None:- by_file.setdefault(row.file_path, []).append((key, _label(row), row.start_line))+ span = (row.start_line, max(row.end_line or 0, row.start_line))+ by_file.setdefault(row.file_path, []).append((key, _label(row), span))+ wanted: dict[str, list[tuple[int, int]]] = {}for hit in rows:- ranked = sorted(by_file.get(targets.get(id(hit), ""), ()), key=lambda r: r[0])+ path = targets.get(id(hit), "")+ ranked = sorted(by_file.get(path, ()), key=lambda r: r[0])if ranked:- hit["symbols"] = [f"{name}:{line}" for _, name, line in ranked[:_MAX_HIT_SYMBOLS]]+ hit["symbols"] = [f"{name}:{span[0]}" for _, name, span in ranked[:_MAX_HIT_SYMBOLS]]+ if path and "matched_lines" not in hit:+ wanted.setdefault(path, [span for *_, span in ranked])+ if not wanted:+ return+ root = Path(repo.local_path or ctx.path)+ sampled = await asyncio.to_thread(_sample_lines, root, list(wanted.items()), terms)+ for hit in rows:+ if found := sampled.get(targets.get(id(hit), "")):+ hit["matched_lines"] = founddiff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pyindex 20344d935..e919ac5a2 100644--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py@@ -1179,7 +1179,8 @@ async def search_codebase(unless the query is why-shaped.Rows naming a file carry `path`; concept pages add `symbols`- (name:line) the query matches. `candidates` lists up to `limit`+ (name:line) the query matches; the top 3 add `matched_lines`+ (line, text), sample lines sharing its words. `candidates` lists up to `limit`distinct files to Read, best first. Identifier or literal queries alsoreturn `lines` (path, line, kind, text; definitions first) read from livefiles; when `complete` is true they are every match, so no grep is needed.diff --git a/tests/unit/server/mcp/test_search_prose_lines.py b/tests/unit/server/mcp/test_search_prose_lines.pynew file mode 100644index 000000000..a8d692cc0--- /dev/null+++ b/tests/unit/server/mcp/test_search_prose_lines.py@@ -0,0 +1,237 @@+"""Concept search rows carry the live source lines that best match a prose query."""++from __future__ import annotations++import pytest++from tests.unit.server.mcp.test_search import _mk_result, _seed_page++TS = """import { pendingRequests } from "./queue";+// pending requests are kept as exact identifiers+export function auditSummary(summary: string): boolean {+ const kept = pendingRequests.every((id) => summary.includes(id));+ return kept && checkIdentifiers(summary);+}+"""++PY = """import os+++def backoff_schedule(polls):+ # schedule for repeated polls+ return POLL_DELAYS[min(polls, len(POLL_DELAYS) - 1)]+++def unrelated():+ return os.getcwd()+"""++GO = """package recovery++// Resume interrupted sessions.+func ResumeOrphanedSession(s *Session) error {+ return s.Send("continue the interrupted session")+}+"""+++def _write(root, rel, text):+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text, encoding="utf-8")+++async def _seed_symbol(session, rid, path, name, start, end, language):+ from repowise.core.persistence.models import WikiSymbol++ session.add(+ WikiSymbol(+ id=f"prose-{path}-{name}",+ repository_id=rid,+ file_path=path,+ symbol_id=f"{path}::{name}",+ name=name,+ qualified_name=name,+ kind="function",+ signature=name,+ start_line=start,+ end_line=end,+ visibility="public",+ language=language,+ )+ )+ await session.commit()+++def _terms(query):+ from repowise.server.mcp_server._hit_symbols import _stem+ from repowise.server.mcp_server._query_terms import content_terms++ return {_stem(t) for t in content_terms(query)}+++class TestBestLines:+ def test_camel_case_lines_match_prose_words(self):+ from repowise.server.mcp_server._hit_symbols import best_lines++ lines = TS.splitlines()+ got = best_lines(lines, _terms("summary keeps pending requests"), [])+ # The import and the comment match too, but a code line outranks them.+ assert [r["line"] for r in got] == [4]+ assert got[0]["text"].startswith("const kept = pendingRequests")++ def test_snake_case_and_symbol_span(self):+ from repowise.server.mcp_server._hit_symbols import best_lines++ lines = PY.splitlines()+ got = best_lines(lines, _terms("schedule of repeated polls"), [(4, 6)])+ assert got == [{"line": 4, "text": "def backoff_schedule(polls):"}]++ def test_comment_lines_serve_when_nothing_else_matches(self):+ from repowise.server.mcp_server._hit_symbols import best_lines++ lines = GO.splitlines()+ terms = _terms("resume interrupted session")+ assert [r["line"] for r in best_lines(lines, terms, [])] == [4, 5]+ # A tie goes to the line inside a matching symbol.+ assert [r["line"] for r in best_lines(lines, terms, [(5, 6)])] == [5, 4]+ only_comment = best_lines(["// resume interrupted work", "x := 1"], _terms("resume interrupted"), [])+ assert only_comment == [{"line": 1, "text": "// resume interrupted work"}]++ def test_long_lines_are_cut_like_line_hits(self):+ from repowise.server.mcp_server._edit_sites import MAX_TEXT_CHARS+ from repowise.server.mcp_server._hit_symbols import best_lines++ got = best_lines(["pending requests " + "x" * 400], _terms("pending requests"), [])+ assert len(got[0]["text"]) == MAX_TEXT_CHARS+++class TestAttachLines:+ @pytest.mark.asyncio+ async def test_top_rows_get_numbered_lines(self, session, populated_db, setup_mcp, tmp_path):+ from repowise.server.mcp_server._helpers import _resolve_repo_context+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols++ _write(tmp_path, "pkg/poll.py", PY)+ _write(tmp_path, "pkg/recovery.go", GO)+ await _seed_symbol(session, populated_db, "pkg/poll.py", "backoff_schedule", 4, 6, "python")+ rows = [+ {"page_type": "file_page", "target_path": "pkg/recovery.go"},+ {"page_type": "file_page", "target_path": "pkg/poll.py"},+ {"page_type": "file_page", "target_path": "pkg/missing.py"},+ ]+ ctx = await _resolve_repo_context(None)+ await attach_hit_symbols(ctx, "resume the interrupted session schedule polls", rows)+ assert rows[0]["matched_lines"] == [+ {"line": 4, "text": "func ResumeOrphanedSession(s *Session) error {"},+ {"line": 5, "text": 'return s.Send("continue the interrupted session")'},+ ]+ assert rows[1]["matched_lines"] == [{"line": 4, "text": "def backoff_schedule(polls):"}]+ # An unreadable file costs only its own field.+ assert "matched_lines" not in rows[2]++ @pytest.mark.asyncio+ async def test_stopword_query_adds_nothing(self, session, populated_db, setup_mcp, tmp_path):+ from repowise.server.mcp_server._helpers import _resolve_repo_context+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols++ _write(tmp_path, "pkg/poll.py", PY)+ rows = [{"page_type": "file_page", "target_path": "pkg/poll.py"}]+ ctx = await _resolve_repo_context(None)+ await attach_hit_symbols(ctx, "what is the", rows)+ assert rows == [{"page_type": "file_page", "target_path": "pkg/poll.py"}]++ @pytest.mark.asyncio+ async def test_concept_search_rows_carry_lines(+ self, session, populated_db, setup_mcp, tmp_path+ ):+ import repowise.server.mcp_server as mcp_mod+ from repowise.server.mcp_server import search_codebase++ _write(tmp_path, "src/audit.ts", TS)+ await _seed_page("file_page:src/audit.ts", "src/audit.ts")++ async def fake_search(query, limit=10):+ return [_mk_result("file_page:src/audit.ts", "Audit", "file_page", "src/audit.ts", 0.9)]++ mcp_mod._vector_store.search = fake_search+ res = await search_codebase(+ "which file checks the summary keeps pending requests", mode="concept"+ )+ row = next(r for r in res["results"] if r.get("path") == "src/audit.ts")+ assert row["matched_lines"][0]["line"] == 4++ @pytest.mark.asyncio+ async def test_identifier_queries_are_unchanged(+ self, session, populated_db, setup_mcp, tmp_path, monkeypatch+ ):+ from repowise.server.mcp_server import _hit_symbols, search_codebase++ _write(tmp_path, "src/audit.ts", TS)+ before = await search_codebase("auditSummary")++ calls = []+ monkeypatch.setattr(_hit_symbols, "best_lines", lambda *a: calls.append(a) or [])+ after = await search_codebase("auditSummary")+ before.pop("_meta", None)+ after.pop("_meta", None)+ assert before == after+ assert calls == []+ assert not any("matched_lines" in r for r in after.get("results", []))+++class TestLineBudget:+ @pytest.mark.asyncio+ async def test_unreadable_files_do_not_spend_the_budget(+ self, session, populated_db, setup_mcp, tmp_path+ ):+ from repowise.server.mcp_server._helpers import _resolve_repo_context+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols++ for name in ("a", "b", "c", "d"):+ _write(tmp_path, f"pkg/{name}.go", GO)+ paths = ["pkg/gone1.go", "pkg/gone2.go", "pkg/a.go", "pkg/b.go", "pkg/c.go", "pkg/d.go"]+ rows = [{"page_type": "file_page", "target_path": p} for p in paths]+ ctx = await _resolve_repo_context(None)+ await attach_hit_symbols(ctx, "resume interrupted session", rows)+ assert [r["target_path"] for r in rows if "matched_lines" in r] == paths[2:5]++ @pytest.mark.asyncio+ async def test_a_hybrid_search_reads_at_most_three_files(+ self, session, populated_db, setup_mcp, tmp_path, monkeypatch+ ):+ import repowise.server.mcp_server as mcp_mod+ from repowise.server.mcp_server import _hit_symbols, search_codebase++ paths = [f"pkg/r{i}.go" for i in range(5)]+ for path in paths:+ _write(tmp_path, path, GO)+ await _seed_page(f"file_page:{path}", path)++ async def fake_search(query, limit=10):+ return [+ _mk_result(f"file_page:{p}", p, "file_page", p, 0.9 - i / 10)+ for i, p in enumerate(paths)+ ]++ mcp_mod._vector_store.search = fake_search+ reads = []+ real = _hit_symbols._read_lines++ def counting(root, rel):+ reads.append(rel)+ return real(root, rel)++ monkeypatch.setattr(_hit_symbols, "_read_lines", counting)+ res = await search_codebase(+ "where is ResumeOrphanedSession resuming the interrupted session", mode="hybrid"+ )+ assert 0 < len(reads) <= 3+ assert sum("matched_lines" in r for r in res["results"]) <= 3+++def test_form_feed_keeps_line_numbers(tmp_path):+ from repowise.server.mcp_server._edit_sites import _read_lines++ (tmp_path / "f.py").write_bytes(b"a = 1\n\x0c\nb = 2\r\nc = 3\n")+ assert _read_lines(tmp_path, "f.py") == ["a = 1", "\x0c", "b = 2", "c = 3"]