Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(search): start snippets at the page text, not its heading (#3192)
Raghav Chamadiyabec4ec0a469a
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex d8a0cfe01..9a6310ca9 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -17,6 +17,7 @@ Usage::from __future__ import annotationsimport logging+import refrom collections import OrderedDictfrom collections.abc import Iterable, Sequencefrom dataclasses import dataclass@@ -51,6 +52,9 @@ class SearchResult:_SNIPPET_LEN = 200+# Markdown heading lines and blank lines at the start of a text.+_LEADING_HEADINGS = re.compile(r"\A(?:[ \t]*(?:#{1,6}(?:[ \t][^\n]*)?)?[ \t]*\r?\n)+")+# Every column of ``page_fts``, in table order. ``page_id`` is stored but not# indexed — it is the join key, never a thing to match on.#@@ -236,6 +240,19 @@ def snippet_around(content: str, query: str, length: int = _SNIPPET_LEN) -> str:return content[start : start + length].strip()+def strip_leading_headings(text: str) -> str:+ """*text* without its leading markdown heading and blank lines.++ For a served snippet only: a page's heading repeats the path and title its+ row already carries. Ranking reads the snippet as built, so callers strip+ after every score is final. Text that is only headings comes back as is.+ """+ match = _LEADING_HEADINGS.match(text)+ if match is None or not text[match.end() :].strip():+ return text+ return text[match.end() :]++class FullTextSearch:"""Full-text search backed by SQLite FTS5 or PostgreSQL tsvector."""diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.pyindex 54e70a042..cfdeaed38 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py@@ -17,6 +17,7 @@ from sqlalchemy import selectfrom repowise.core.persistence.database import get_sessionfrom repowise.core.persistence.models import Page+from repowise.core.persistence.search import strip_leading_headingsfrom repowise.server.mcp_server._page_paths import hit_file_pathfrom repowise.server.mcp_server._query_terms import content_termsfrom repowise.server.mcp_server._retrieval_rank import rerank_by_context_coverage@@ -183,6 +184,8 @@ def serialize_hits(for key in ("snippet", "excerpt"):if h.get(key) and (key != "excerpt" or serve_excerpt):entry[key] = h[key]+ if entry.get("snippet"):+ entry["snippet"] = strip_leading_headings(entry["snippet"])if h.get("score") is not None:entry["score"] = round(h["score"], 3)expanded = "graph_expand" in (h.get("_sources") or ())diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.pyindex e919ac5a2..0a770b547 100644--- a/packages/server/src/repowise/server/mcp_server/tool_search.py+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py@@ -16,6 +16,7 @@ from repowise.core.persistence.models import (GitMetadata,Page,)+from repowise.core.persistence.search import strip_leading_headingsfrom repowise.core.providers.embedding import store_has_semantic_vectorsfrom repowise.core.registry import ToolRecipefrom repowise.core.registry import mcp_tool_registry as mcp@@ -576,6 +577,13 @@ def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:return results+def _serve_snippets(results: list[dict]) -> None:+ """Drop each snippet's leading page heading, after ranking has read it."""+ for item in results:+ if item.get("snippet"):+ item["snippet"] = strip_leading_headings(item["snippet"])++def _drop_internal_ranking_fields(results: list[dict]) -> None:"""Keep calibration diagnostics internal to the ranking pipeline."""for item in results:@@ -834,6 +842,7 @@ async def _federated_search(# Last, so nothing above has to know the field is on its way out.add_row_paths(output)_drop_derivable_page_ids(output)+ _serve_snippets(output)return response@@ -1139,6 +1148,7 @@ async def _structured_search(# first, so a page whose target_path is dropped keeps its page_id.add_row_paths(results)_drop_derivable_page_ids(results)+ _serve_snippets(results)return response@@ -1339,4 +1349,5 @@ async def search_codebase(# Last, so nothing above has to know the field is on its way out.add_row_paths(output)_drop_derivable_page_ids(output)+ _serve_snippets(output)return responsediff --git a/tests/unit/persistence/test_evidence_snippet.py b/tests/unit/persistence/test_evidence_snippet.pyindex 7d78843f3..312d10964 100644--- a/tests/unit/persistence/test_evidence_snippet.py+++ b/tests/unit/persistence/test_evidence_snippet.py@@ -14,7 +14,11 @@ from __future__ import annotationsimport pytest-from repowise.core.persistence.search import FullTextSearch, snippet_around+from repowise.core.persistence.search import (+ FullTextSearch,+ snippet_around,+ strip_leading_headings,+)_OPENER = "## Overview\n\nThis module is part of the indexing pipeline. " + "Filler. " * 40_MATCH = "The walker refuses to descend into a nested git checkout."@@ -90,6 +94,43 @@ class TestSnippetAround:assert snippet_around(content, "the").startswith("## Overview")+class TestStripLeadingHeadings:+ """A served snippet drops the heading that repeats its row's path."""++ _BODY = "Loads persisted session history for reseeding."++ @pytest.mark.parametrize(+ "head",+ [+ "# src/agents/session_history.ts\n\n## Overview\n\n",+ "# pkg/walker.go\n\n",+ "## Overview\n",+ "# Module: billing\r\n\r\n### Summary\r\n\r\n",+ "\n\n# Decision: use one store\n \n",+ ],+ )+ def test_leading_headings_are_dropped(self, head):+ assert strip_leading_headings(head + self._BODY) == self._BODY++ def test_text_without_headings_is_unchanged(self):+ assert strip_leading_headings(self._BODY) == self._BODY++ def test_a_heading_after_text_is_kept(self):+ text = "Intro prose.\n## Nested checkout\nTail."+ assert strip_leading_headings(text) == text++ def test_hash_without_a_space_is_not_a_heading(self):+ text = "#include <walker.h>\nint main(void);"+ assert strip_leading_headings(text) == text++ def test_text_of_only_headings_is_returned_as_is(self):+ text = "# src/empty.py\n\n## Overview\n"+ assert strip_leading_headings(text) == text++ def test_empty_text_stays_empty(self):+ assert strip_leading_headings("") == ""++class TestFullTextSnippet:async def test_a_full_text_hit_shows_the_matching_region(self, fts):await fts.index(diff --git a/tests/unit/server/mcp/test_answer_payload.py b/tests/unit/server/mcp/test_answer_payload.pyindex 6f9d36d08..58a6cbf6e 100644--- a/tests/unit/server/mcp/test_answer_payload.py+++ b/tests/unit/server/mcp/test_answer_payload.py@@ -145,6 +145,16 @@ class TestSerializeHits:[entry] = serialize_hits([dict(RAW_HIT)], summary_chars=160)assert len(entry["summary"]) <= 160+ def test_snippet_starts_below_the_page_heading(self) -> None:+ hit = dict(+ RAW_HIT,+ snippet="# pkg/mod.py\n\n## Overview\n\nParses the config.",+ excerpt="# pkg/mod.py\n\nBody.",+ )+ [entry] = serialize_hits([hit])+ assert entry["snippet"] == "Parses the config."+ assert entry["excerpt"] == "# pkg/mod.py\n\nBody."+def test_limit(self) -> None:hits = [dict(RAW_HIT), dict(RAW_HIT), dict(RAW_HIT)]assert len(serialize_hits(hits, limit=2)) == 2diff --git a/tests/unit/server/mcp/test_search.py b/tests/unit/server/mcp/test_search.pyindex de59a51c8..a6ca927bc 100644--- a/tests/unit/server/mcp/test_search.py+++ b/tests/unit/server/mcp/test_search.py@@ -8,6 +8,8 @@ from __future__ import annotationsimport pytest+from repowise.core.persistence.search import strip_leading_headings+@pytest.mark.asyncioasync def test_search_codebase(setup_mcp):@@ -2252,3 +2254,69 @@ class TestHitSymbols:"services/mixer.py",]assert not any("symbols" in r for r in res["results"])+++_SERVED_SNIPPETS = {+ "src/session/cache.py": "# src/session/cache.py\n\n## Overview\n\nKeeps session rows warm.",+ "pkg/store.go": "# pkg/store.go\r\n\r\nSession store backed by a map.",+ "lib/layer.rb": "Cache layer with no heading. ## not a heading here",+ "docs/only.md": "# docs/only.md\n\n## Overview\n",+}+++class TestServedSnippet:+ """A served snippet starts at the page text; ranking still reads the heading."""++ async def _search(self, monkeypatch, strip):+ import repowise.server.mcp_server as mcp_mod+ from repowise.core.persistence.search import SearchResult+ from repowise.server.mcp_server import search_codebase, tool_search++ monkeypatch.setattr(tool_search, "strip_leading_headings", strip)++ def hits(scale):+ return [+ SearchResult(+ page_id=f"file_page:{path}",+ title=path,+ page_type="file_page",+ target_path=path,+ score=scale * (4 - i),+ snippet=snippet,+ search_type="fulltext",+ )+ for i, (path, snippet) in enumerate(_SERVED_SNIPPETS.items())+ ]++ async def fake_vec(query, limit=10):+ return hits(0.2)++ async def fake_fts(query, limit=10):+ return hits(1.0)++ monkeypatch.setattr(mcp_mod._vector_store, "search", fake_vec)+ monkeypatch.setattr(mcp_mod._fts, "search", fake_fts)+ return await search_codebase("session cache layer", limit=10, mode="concept")++ @pytest.mark.asyncio+ async def test_rows_match_the_unstripped_search_except_the_snippet(+ self, setup_mcp, monkeypatch+ ):+ for path in _SERVED_SNIPPETS:+ await _seed_page(f"file_page:{path}", path)++ raw = await self._search(monkeypatch, lambda text: text)+ served = await self._search(monkeypatch, strip_leading_headings)++ def without_snippet(rows):+ return [{k: v for k, v in row.items() if k != "snippet"} for row in rows]++ assert without_snippet(served["results"]) == without_snippet(raw["results"])+ assert served.get("candidates") == raw.get("candidates")+ snippets = {row["path"]: row["snippet"] for row in served["results"]}+ assert snippets == {+ "src/session/cache.py": "Keeps session rows warm.",+ "pkg/store.go": "Session store backed by a map.",+ "lib/layer.rb": "Cache layer with no heading. ## not a heading here",+ "docs/only.md": "# docs/only.md\n\n## Overview\n",+ }