Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(search): return the best-matching lines for prose queries (#3189)

Raghav Chamadiyae53e753370bf
diff --git a/docs/agent/MCP_TOOLS.md b/docs/agent/MCP_TOOLS.md
index 2680d51ba..5526c5c4b 100644
--- a/docs/agent/MCP_TOOLS.md
+++ b/docs/agent/MCP_TOOLS.md
@@ -223,7 +223,7 @@ Hybrid search that routes by the shape of the query: identifiers search the symb
**Key return fields:** `results` (every row that names a file carries it in `path`, and a row naming no file has no `path`; symbol hits carry `symbol_id`, line bounds and `signature`, plus `symbols` (`name:line` of up to five other matches in that file, then `+N more`) when several matched; concept hits carry `relevance_score`, `snippet` and `sources`; `file` on symbol and file hits is a deprecated alias of `path`, removed in the next minor release, and a page keeps `target_path` only where it differs from `path`), `candidates` (up to `limit` distinct openable file paths, best first). If your next move is a Read, read `candidates`: some `results` are pages that are not files.
-**Page rows** also carry `symbols`: up to three `name:line` entries (`Owner.member` for members) in that file whose names match the query's words.
+**Page rows** also carry `symbols`: up to three `name:line` entries (`Owner.member` for members) in that file whose names match the query's words. The first three page rows whose file has a matching line also carry `matched_lines`: a sample of up to three `{line, text}` source lines read from the live file that share the most query words, code lines before comments and imports. Unlike `lines`, it is never every match.
An identifier or literal query also returns `lines` (`{path, line, kind, text}`, definitions first, at most 50): the `get_context` `references` of one to three exact symbols, else a live scan for the token (kind `definition` or `match`). `complete` is true only when they are every match (see `references` above); otherwise `reasons` says why.
diff --git a/packages/server/src/repowise/server/mcp_server/_edit_sites.py b/packages/server/src/repowise/server/mcp_server/_edit_sites.py
index c54dfcb1e..d082cde41 100644
--- a/packages/server/src/repowise/server/mcp_server/_edit_sites.py
+++ b/packages/server/src/repowise/server/mcp_server/_edit_sites.py
@@ -84,7 +84,11 @@ def _read_lines(root: Path, rel: str) -> list[str] | None:
except OSError:
return None
text = read_repo_file_text(root, rel)
- return None if text is None else text.splitlines()
+ if text is None:
+ return None
+ # Only a newline ends a line: splitlines() also splits on form feeds and
+ # other separators, which would shift every line number after them.
+ return [line.removesuffix("\r") for line in text.removesuffix("\n").split("\n")]
def _scan(
diff --git a/packages/server/src/repowise/server/mcp_server/_hit_symbols.py b/packages/server/src/repowise/server/mcp_server/_hit_symbols.py
index b5743af98..e408e3b83 100644
--- a/packages/server/src/repowise/server/mcp_server/_hit_symbols.py
+++ b/packages/server/src/repowise/server/mcp_server/_hit_symbols.py
@@ -4,17 +4,23 @@ A search row names a file; the caller usually wants the function in it next.
Each file-backed page row gets ``symbols``: up to :data:`_MAX_HIT_SYMBOLS`
entries of ``name:line``, the same shape a collapsed symbol row already carries
for its same-file neighbours. A member is named ``Owner.member`` so a bare
-``run:40`` is never ambiguous.
+``run:40`` is never ambiguous. The first :data:`_MAX_LINE_FILES` rows whose live
+file has a line sharing query words also get ``matched_lines``: a sample of the
+best such lines, not every match.
"""
from __future__ import annotations
+import asyncio
import logging
+import re
+from pathlib import Path
from sqlalchemy import select
from repowise.core.persistence.database import get_session
from repowise.core.persistence.models import WikiSymbol
+from repowise.server.mcp_server._edit_sites import MAX_TEXT_CHARS, _import_lines, _read_lines
from repowise.server.mcp_server._helpers import _get_repo
from repowise.server.mcp_server._page_paths import hit_file_path
from repowise.server.mcp_server._query_terms import content_terms
@@ -23,6 +29,12 @@ from repowise.server.mcp_server.tool_search_symbols import _tokens
_log = logging.getLogger("repowise.mcp.search")
_MAX_HIT_SYMBOLS = 3
+# Only the top files are read: past them an agent opens the file anyway.
+_MAX_LINE_FILES = 3
+_MAX_HIT_LINES = 3
+# Files tried for lines, so unreadable or unmatched ones cannot make it unbounded.
+_MAX_LINE_READS = 2 * _MAX_LINE_FILES
+_COMMENT_ONLY = re.compile(r"^\s*(?:#|//|/\*|\*|--|<!--)")
# Values rank after the callables and types a navigation question is after.
_VALUE_KINDS = frozenset({"constant", "variable", "property"})
@@ -73,12 +85,51 @@ def _label(row) -> str:
return row.name
+def best_lines(
+ lines: list[str], terms: set[str], spans: list[tuple[int, int]]
+) -> list[dict]:
+ """``{line, text}`` for the lines sharing the most of *terms*, best first.
+
+ A line inside a symbol whose name matches wins a tie; comment-only and
+ import lines are served only when no other line matches. With two or more
+ terms a line must match two, so one common word does not decide alone.
+ """
+ need = min(2, len(terms))
+ imports = _import_lines(lines)
+ scored = []
+ for n, text in enumerate(lines, 1):
+ hits = len(terms & _stems(text))
+ if hits < need:
+ continue
+ weak = n in imports or bool(_COMMENT_ONLY.match(text))
+ inside = any(start <= n <= end for start, end in spans)
+ scored.append((weak, -hits, not inside, n, text))
+ if any(not s[0] for s in scored):
+ scored = [s for s in scored if not s[0]]
+ best = sorted(scored)[:_MAX_HIT_LINES]
+ return [{"line": n, "text": text.strip()[:MAX_TEXT_CHARS]} for *_, n, text in best]
+
+
+def _sample_lines(
+ root: Path, wanted: list[tuple[str, list[tuple[int, int]]]], terms: set[str]
+) -> dict[str, list[dict]]:
+ """Best lines for the first files in *wanted* that yield any, read in order."""
+ found: dict[str, list[dict]] = {}
+ for path, spans in wanted[:_MAX_LINE_READS]:
+ lines = _read_lines(root, path)
+ if lines and (best := best_lines(lines, terms, spans)):
+ found[path] = best
+ if len(found) >= _MAX_LINE_FILES:
+ break
+ return found
+
+
async def attach_hit_symbols(ctx, query: str, rows: list[dict]) -> None:
- """Add ``symbols`` to each file-backed page row, in place; one query for all rows.
+ """Add ``symbols`` (and ``matched_lines`` to top rows) to file-backed page rows, in place.
- Symbol rows are left alone (they already name their symbol), and a row whose
- file has no symbol sharing a query word gets no field. A failure here only
- costs the field, never the search.
+ One symbol query serves all rows. Symbol rows are left alone (they already
+ name their symbol), and a row with nothing sharing a query word gets no
+ field. A failure here only costs the fields, never the search.
"""
try:
await _attach(ctx, query, rows)
@@ -106,6 +157,7 @@ async def _attach(ctx, query: str, rows: list[dict]) -> None:
WikiSymbol.parent_name,
WikiSymbol.kind,
WikiSymbol.start_line,
+ WikiSymbol.end_line,
WikiSymbol.signature,
WikiSymbol.docstring,
WikiSymbol.visibility,
@@ -119,8 +171,20 @@ async def _attach(ctx, query: str, rows: list[dict]) -> None:
for row in result.all():
key = _rank_key(row, terms)
if key is not None:
- by_file.setdefault(row.file_path, []).append((key, _label(row), row.start_line))
+ span = (row.start_line, max(row.end_line or 0, row.start_line))
+ by_file.setdefault(row.file_path, []).append((key, _label(row), span))
+ wanted: dict[str, list[tuple[int, int]]] = {}
for hit in rows:
- ranked = sorted(by_file.get(targets.get(id(hit), ""), ()), key=lambda r: r[0])
+ path = targets.get(id(hit), "")
+ ranked = sorted(by_file.get(path, ()), key=lambda r: r[0])
if ranked:
- hit["symbols"] = [f"{name}:{line}" for _, name, line in ranked[:_MAX_HIT_SYMBOLS]]
+ hit["symbols"] = [f"{name}:{span[0]}" for _, name, span in ranked[:_MAX_HIT_SYMBOLS]]
+ if path and "matched_lines" not in hit:
+ wanted.setdefault(path, [span for *_, span in ranked])
+ if not wanted:
+ return
+ root = Path(repo.local_path or ctx.path)
+ sampled = await asyncio.to_thread(_sample_lines, root, list(wanted.items()), terms)
+ for hit in rows:
+ if found := sampled.get(targets.get(id(hit), "")):
+ hit["matched_lines"] = found
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.py
index 20344d935..e919ac5a2 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_search.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py
@@ -1179,7 +1179,8 @@ async def search_codebase(
unless the query is why-shaped.
Rows naming a file carry `path`; concept pages add `symbols`
- (name:line) the query matches. `candidates` lists up to `limit`
+ (name:line) the query matches; the top 3 add `matched_lines`
+ (line, text), sample lines sharing its words. `candidates` lists up to `limit`
distinct files to Read, best first. Identifier or literal queries also
return `lines` (path, line, kind, text; definitions first) read from live
files; when `complete` is true they are every match, so no grep is needed.
diff --git a/tests/unit/server/mcp/test_search_prose_lines.py b/tests/unit/server/mcp/test_search_prose_lines.py
new file mode 100644
index 000000000..a8d692cc0
--- /dev/null
+++ b/tests/unit/server/mcp/test_search_prose_lines.py
@@ -0,0 +1,237 @@
+"""Concept search rows carry the live source lines that best match a prose query."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.unit.server.mcp.test_search import _mk_result, _seed_page
+
+TS = """import { pendingRequests } from "./queue";
+// pending requests are kept as exact identifiers
+export function auditSummary(summary: string): boolean {
+ const kept = pendingRequests.every((id) => summary.includes(id));
+ return kept && checkIdentifiers(summary);
+}
+"""
+
+PY = """import os
+
+
+def backoff_schedule(polls):
+ # schedule for repeated polls
+ return POLL_DELAYS[min(polls, len(POLL_DELAYS) - 1)]
+
+
+def unrelated():
+ return os.getcwd()
+"""
+
+GO = """package recovery
+
+// Resume interrupted sessions.
+func ResumeOrphanedSession(s *Session) error {
+ return s.Send("continue the interrupted session")
+}
+"""
+
+
+def _write(root, rel, text):
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text, encoding="utf-8")
+
+
+async def _seed_symbol(session, rid, path, name, start, end, language):
+ from repowise.core.persistence.models import WikiSymbol
+
+ session.add(
+ WikiSymbol(
+ id=f"prose-{path}-{name}",
+ repository_id=rid,
+ file_path=path,
+ symbol_id=f"{path}::{name}",
+ name=name,
+ qualified_name=name,
+ kind="function",
+ signature=name,
+ start_line=start,
+ end_line=end,
+ visibility="public",
+ language=language,
+ )
+ )
+ await session.commit()
+
+
+def _terms(query):
+ from repowise.server.mcp_server._hit_symbols import _stem
+ from repowise.server.mcp_server._query_terms import content_terms
+
+ return {_stem(t) for t in content_terms(query)}
+
+
+class TestBestLines:
+ def test_camel_case_lines_match_prose_words(self):
+ from repowise.server.mcp_server._hit_symbols import best_lines
+
+ lines = TS.splitlines()
+ got = best_lines(lines, _terms("summary keeps pending requests"), [])
+ # The import and the comment match too, but a code line outranks them.
+ assert [r["line"] for r in got] == [4]
+ assert got[0]["text"].startswith("const kept = pendingRequests")
+
+ def test_snake_case_and_symbol_span(self):
+ from repowise.server.mcp_server._hit_symbols import best_lines
+
+ lines = PY.splitlines()
+ got = best_lines(lines, _terms("schedule of repeated polls"), [(4, 6)])
+ assert got == [{"line": 4, "text": "def backoff_schedule(polls):"}]
+
+ def test_comment_lines_serve_when_nothing_else_matches(self):
+ from repowise.server.mcp_server._hit_symbols import best_lines
+
+ lines = GO.splitlines()
+ terms = _terms("resume interrupted session")
+ assert [r["line"] for r in best_lines(lines, terms, [])] == [4, 5]
+ # A tie goes to the line inside a matching symbol.
+ assert [r["line"] for r in best_lines(lines, terms, [(5, 6)])] == [5, 4]
+ only_comment = best_lines(["// resume interrupted work", "x := 1"], _terms("resume interrupted"), [])
+ assert only_comment == [{"line": 1, "text": "// resume interrupted work"}]
+
+ def test_long_lines_are_cut_like_line_hits(self):
+ from repowise.server.mcp_server._edit_sites import MAX_TEXT_CHARS
+ from repowise.server.mcp_server._hit_symbols import best_lines
+
+ got = best_lines(["pending requests " + "x" * 400], _terms("pending requests"), [])
+ assert len(got[0]["text"]) == MAX_TEXT_CHARS
+
+
+class TestAttachLines:
+ @pytest.mark.asyncio
+ async def test_top_rows_get_numbered_lines(self, session, populated_db, setup_mcp, tmp_path):
+ from repowise.server.mcp_server._helpers import _resolve_repo_context
+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+
+ _write(tmp_path, "pkg/poll.py", PY)
+ _write(tmp_path, "pkg/recovery.go", GO)
+ await _seed_symbol(session, populated_db, "pkg/poll.py", "backoff_schedule", 4, 6, "python")
+ rows = [
+ {"page_type": "file_page", "target_path": "pkg/recovery.go"},
+ {"page_type": "file_page", "target_path": "pkg/poll.py"},
+ {"page_type": "file_page", "target_path": "pkg/missing.py"},
+ ]
+ ctx = await _resolve_repo_context(None)
+ await attach_hit_symbols(ctx, "resume the interrupted session schedule polls", rows)
+ assert rows[0]["matched_lines"] == [
+ {"line": 4, "text": "func ResumeOrphanedSession(s *Session) error {"},
+ {"line": 5, "text": 'return s.Send("continue the interrupted session")'},
+ ]
+ assert rows[1]["matched_lines"] == [{"line": 4, "text": "def backoff_schedule(polls):"}]
+ # An unreadable file costs only its own field.
+ assert "matched_lines" not in rows[2]
+
+ @pytest.mark.asyncio
+ async def test_stopword_query_adds_nothing(self, session, populated_db, setup_mcp, tmp_path):
+ from repowise.server.mcp_server._helpers import _resolve_repo_context
+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+
+ _write(tmp_path, "pkg/poll.py", PY)
+ rows = [{"page_type": "file_page", "target_path": "pkg/poll.py"}]
+ ctx = await _resolve_repo_context(None)
+ await attach_hit_symbols(ctx, "what is the", rows)
+ assert rows == [{"page_type": "file_page", "target_path": "pkg/poll.py"}]
+
+ @pytest.mark.asyncio
+ async def test_concept_search_rows_carry_lines(
+ self, session, populated_db, setup_mcp, tmp_path
+ ):
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server import search_codebase
+
+ _write(tmp_path, "src/audit.ts", TS)
+ await _seed_page("file_page:src/audit.ts", "src/audit.ts")
+
+ async def fake_search(query, limit=10):
+ return [_mk_result("file_page:src/audit.ts", "Audit", "file_page", "src/audit.ts", 0.9)]
+
+ mcp_mod._vector_store.search = fake_search
+ res = await search_codebase(
+ "which file checks the summary keeps pending requests", mode="concept"
+ )
+ row = next(r for r in res["results"] if r.get("path") == "src/audit.ts")
+ assert row["matched_lines"][0]["line"] == 4
+
+ @pytest.mark.asyncio
+ async def test_identifier_queries_are_unchanged(
+ self, session, populated_db, setup_mcp, tmp_path, monkeypatch
+ ):
+ from repowise.server.mcp_server import _hit_symbols, search_codebase
+
+ _write(tmp_path, "src/audit.ts", TS)
+ before = await search_codebase("auditSummary")
+
+ calls = []
+ monkeypatch.setattr(_hit_symbols, "best_lines", lambda *a: calls.append(a) or [])
+ after = await search_codebase("auditSummary")
+ before.pop("_meta", None)
+ after.pop("_meta", None)
+ assert before == after
+ assert calls == []
+ assert not any("matched_lines" in r for r in after.get("results", []))
+
+
+class TestLineBudget:
+ @pytest.mark.asyncio
+ async def test_unreadable_files_do_not_spend_the_budget(
+ self, session, populated_db, setup_mcp, tmp_path
+ ):
+ from repowise.server.mcp_server._helpers import _resolve_repo_context
+ from repowise.server.mcp_server._hit_symbols import attach_hit_symbols
+
+ for name in ("a", "b", "c", "d"):
+ _write(tmp_path, f"pkg/{name}.go", GO)
+ paths = ["pkg/gone1.go", "pkg/gone2.go", "pkg/a.go", "pkg/b.go", "pkg/c.go", "pkg/d.go"]
+ rows = [{"page_type": "file_page", "target_path": p} for p in paths]
+ ctx = await _resolve_repo_context(None)
+ await attach_hit_symbols(ctx, "resume interrupted session", rows)
+ assert [r["target_path"] for r in rows if "matched_lines" in r] == paths[2:5]
+
+ @pytest.mark.asyncio
+ async def test_a_hybrid_search_reads_at_most_three_files(
+ self, session, populated_db, setup_mcp, tmp_path, monkeypatch
+ ):
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.server.mcp_server import _hit_symbols, search_codebase
+
+ paths = [f"pkg/r{i}.go" for i in range(5)]
+ for path in paths:
+ _write(tmp_path, path, GO)
+ await _seed_page(f"file_page:{path}", path)
+
+ async def fake_search(query, limit=10):
+ return [
+ _mk_result(f"file_page:{p}", p, "file_page", p, 0.9 - i / 10)
+ for i, p in enumerate(paths)
+ ]
+
+ mcp_mod._vector_store.search = fake_search
+ reads = []
+ real = _hit_symbols._read_lines
+
+ def counting(root, rel):
+ reads.append(rel)
+ return real(root, rel)
+
+ monkeypatch.setattr(_hit_symbols, "_read_lines", counting)
+ res = await search_codebase(
+ "where is ResumeOrphanedSession resuming the interrupted session", mode="hybrid"
+ )
+ assert 0 < len(reads) <= 3
+ assert sum("matched_lines" in r for r in res["results"]) <= 3
+
+
+def test_form_feed_keeps_line_numbers(tmp_path):
+ from repowise.server.mcp_server._edit_sites import _read_lines
+
+ (tmp_path / "f.py").write_bytes(b"a = 1\n\x0c\nb = 2\r\nc = 3\n")
+ assert _read_lines(tmp_path, "f.py") == ["a = 1", "\x0c", "b = 2", "c = 3"]