Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(search): start snippets at the page text, not its heading (#3192)

Raghav Chamadiyabec4ec0a469a
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index d8a0cfe01..9a6310ca9 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -17,6 +17,7 @@ Usage::
from __future__ import annotations
import logging
+import re
from collections import OrderedDict
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
@@ -51,6 +52,9 @@ class SearchResult:
_SNIPPET_LEN = 200
+# Markdown heading lines and blank lines at the start of a text.
+_LEADING_HEADINGS = re.compile(r"\A(?:[ \t]*(?:#{1,6}(?:[ \t][^\n]*)?)?[ \t]*\r?\n)+")
+
# Every column of ``page_fts``, in table order. ``page_id`` is stored but not
# indexed — it is the join key, never a thing to match on.
#
@@ -236,6 +240,19 @@ def snippet_around(content: str, query: str, length: int = _SNIPPET_LEN) -> str:
return content[start : start + length].strip()
+def strip_leading_headings(text: str) -> str:
+ """*text* without its leading markdown heading and blank lines.
+
+ For a served snippet only: a page's heading repeats the path and title its
+ row already carries. Ranking reads the snippet as built, so callers strip
+ after every score is final. Text that is only headings comes back as is.
+ """
+ match = _LEADING_HEADINGS.match(text)
+ if match is None or not text[match.end() :].strip():
+ return text
+ return text[match.end() :]
+
+
class FullTextSearch:
"""Full-text search backed by SQLite FTS5 or PostgreSQL tsvector."""
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py
index 54e70a042..cfdeaed38 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/retrieval.py
@@ -17,6 +17,7 @@ from sqlalchemy import select
from repowise.core.persistence.database import get_session
from repowise.core.persistence.models import Page
+from repowise.core.persistence.search import strip_leading_headings
from repowise.server.mcp_server._page_paths import hit_file_path
from repowise.server.mcp_server._query_terms import content_terms
from repowise.server.mcp_server._retrieval_rank import rerank_by_context_coverage
@@ -183,6 +184,8 @@ def serialize_hits(
for key in ("snippet", "excerpt"):
if h.get(key) and (key != "excerpt" or serve_excerpt):
entry[key] = h[key]
+ if entry.get("snippet"):
+ entry["snippet"] = strip_leading_headings(entry["snippet"])
if h.get("score") is not None:
entry["score"] = round(h["score"], 3)
expanded = "graph_expand" in (h.get("_sources") or ())
diff --git a/packages/server/src/repowise/server/mcp_server/tool_search.py b/packages/server/src/repowise/server/mcp_server/tool_search.py
index e919ac5a2..0a770b547 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_search.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_search.py
@@ -16,6 +16,7 @@ from repowise.core.persistence.models import (
GitMetadata,
Page,
)
+from repowise.core.persistence.search import strip_leading_headings
from repowise.core.providers.embedding import store_has_semantic_vectors
from repowise.core.registry import ToolRecipe
from repowise.core.registry import mcp_tool_registry as mcp
@@ -576,6 +577,13 @@ def _drop_derivable_page_ids(results: list[dict]) -> list[dict]:
return results
+def _serve_snippets(results: list[dict]) -> None:
+ """Drop each snippet's leading page heading, after ranking has read it."""
+ for item in results:
+ if item.get("snippet"):
+ item["snippet"] = strip_leading_headings(item["snippet"])
+
+
def _drop_internal_ranking_fields(results: list[dict]) -> None:
"""Keep calibration diagnostics internal to the ranking pipeline."""
for item in results:
@@ -834,6 +842,7 @@ async def _federated_search(
# Last, so nothing above has to know the field is on its way out.
add_row_paths(output)
_drop_derivable_page_ids(output)
+ _serve_snippets(output)
return response
@@ -1139,6 +1148,7 @@ async def _structured_search(
# first, so a page whose target_path is dropped keeps its page_id.
add_row_paths(results)
_drop_derivable_page_ids(results)
+ _serve_snippets(results)
return response
@@ -1339,4 +1349,5 @@ async def search_codebase(
# Last, so nothing above has to know the field is on its way out.
add_row_paths(output)
_drop_derivable_page_ids(output)
+ _serve_snippets(output)
return response
diff --git a/tests/unit/persistence/test_evidence_snippet.py b/tests/unit/persistence/test_evidence_snippet.py
index 7d78843f3..312d10964 100644
--- a/tests/unit/persistence/test_evidence_snippet.py
+++ b/tests/unit/persistence/test_evidence_snippet.py
@@ -14,7 +14,11 @@ from __future__ import annotations
import pytest
-from repowise.core.persistence.search import FullTextSearch, snippet_around
+from repowise.core.persistence.search import (
+ FullTextSearch,
+ snippet_around,
+ strip_leading_headings,
+)
_OPENER = "## Overview\n\nThis module is part of the indexing pipeline. " + "Filler. " * 40
_MATCH = "The walker refuses to descend into a nested git checkout."
@@ -90,6 +94,43 @@ class TestSnippetAround:
assert snippet_around(content, "the").startswith("## Overview")
+class TestStripLeadingHeadings:
+ """A served snippet drops the heading that repeats its row's path."""
+
+ _BODY = "Loads persisted session history for reseeding."
+
+ @pytest.mark.parametrize(
+ "head",
+ [
+ "# src/agents/session_history.ts\n\n## Overview\n\n",
+ "# pkg/walker.go\n\n",
+ "## Overview\n",
+ "# Module: billing\r\n\r\n### Summary\r\n\r\n",
+ "\n\n# Decision: use one store\n \n",
+ ],
+ )
+ def test_leading_headings_are_dropped(self, head):
+ assert strip_leading_headings(head + self._BODY) == self._BODY
+
+ def test_text_without_headings_is_unchanged(self):
+ assert strip_leading_headings(self._BODY) == self._BODY
+
+ def test_a_heading_after_text_is_kept(self):
+ text = "Intro prose.\n## Nested checkout\nTail."
+ assert strip_leading_headings(text) == text
+
+ def test_hash_without_a_space_is_not_a_heading(self):
+ text = "#include <walker.h>\nint main(void);"
+ assert strip_leading_headings(text) == text
+
+ def test_text_of_only_headings_is_returned_as_is(self):
+ text = "# src/empty.py\n\n## Overview\n"
+ assert strip_leading_headings(text) == text
+
+ def test_empty_text_stays_empty(self):
+ assert strip_leading_headings("") == ""
+
+
class TestFullTextSnippet:
async def test_a_full_text_hit_shows_the_matching_region(self, fts):
await fts.index(
diff --git a/tests/unit/server/mcp/test_answer_payload.py b/tests/unit/server/mcp/test_answer_payload.py
index 6f9d36d08..58a6cbf6e 100644
--- a/tests/unit/server/mcp/test_answer_payload.py
+++ b/tests/unit/server/mcp/test_answer_payload.py
@@ -145,6 +145,16 @@ class TestSerializeHits:
[entry] = serialize_hits([dict(RAW_HIT)], summary_chars=160)
assert len(entry["summary"]) <= 160
+ def test_snippet_starts_below_the_page_heading(self) -> None:
+ hit = dict(
+ RAW_HIT,
+ snippet="# pkg/mod.py\n\n## Overview\n\nParses the config.",
+ excerpt="# pkg/mod.py\n\nBody.",
+ )
+ [entry] = serialize_hits([hit])
+ assert entry["snippet"] == "Parses the config."
+ assert entry["excerpt"] == "# pkg/mod.py\n\nBody."
+
def test_limit(self) -> None:
hits = [dict(RAW_HIT), dict(RAW_HIT), dict(RAW_HIT)]
assert len(serialize_hits(hits, limit=2)) == 2
diff --git a/tests/unit/server/mcp/test_search.py b/tests/unit/server/mcp/test_search.py
index de59a51c8..a6ca927bc 100644
--- a/tests/unit/server/mcp/test_search.py
+++ b/tests/unit/server/mcp/test_search.py
@@ -8,6 +8,8 @@ from __future__ import annotations
import pytest
+from repowise.core.persistence.search import strip_leading_headings
+
@pytest.mark.asyncio
async def test_search_codebase(setup_mcp):
@@ -2252,3 +2254,69 @@ class TestHitSymbols:
"services/mixer.py",
]
assert not any("symbols" in r for r in res["results"])
+
+
+_SERVED_SNIPPETS = {
+ "src/session/cache.py": "# src/session/cache.py\n\n## Overview\n\nKeeps session rows warm.",
+ "pkg/store.go": "# pkg/store.go\r\n\r\nSession store backed by a map.",
+ "lib/layer.rb": "Cache layer with no heading. ## not a heading here",
+ "docs/only.md": "# docs/only.md\n\n## Overview\n",
+}
+
+
+class TestServedSnippet:
+ """A served snippet starts at the page text; ranking still reads the heading."""
+
+ async def _search(self, monkeypatch, strip):
+ import repowise.server.mcp_server as mcp_mod
+ from repowise.core.persistence.search import SearchResult
+ from repowise.server.mcp_server import search_codebase, tool_search
+
+ monkeypatch.setattr(tool_search, "strip_leading_headings", strip)
+
+ def hits(scale):
+ return [
+ SearchResult(
+ page_id=f"file_page:{path}",
+ title=path,
+ page_type="file_page",
+ target_path=path,
+ score=scale * (4 - i),
+ snippet=snippet,
+ search_type="fulltext",
+ )
+ for i, (path, snippet) in enumerate(_SERVED_SNIPPETS.items())
+ ]
+
+ async def fake_vec(query, limit=10):
+ return hits(0.2)
+
+ async def fake_fts(query, limit=10):
+ return hits(1.0)
+
+ monkeypatch.setattr(mcp_mod._vector_store, "search", fake_vec)
+ monkeypatch.setattr(mcp_mod._fts, "search", fake_fts)
+ return await search_codebase("session cache layer", limit=10, mode="concept")
+
+ @pytest.mark.asyncio
+ async def test_rows_match_the_unstripped_search_except_the_snippet(
+ self, setup_mcp, monkeypatch
+ ):
+ for path in _SERVED_SNIPPETS:
+ await _seed_page(f"file_page:{path}", path)
+
+ raw = await self._search(monkeypatch, lambda text: text)
+ served = await self._search(monkeypatch, strip_leading_headings)
+
+ def without_snippet(rows):
+ return [{k: v for k, v in row.items() if k != "snippet"} for row in rows]
+
+ assert without_snippet(served["results"]) == without_snippet(raw["results"])
+ assert served.get("candidates") == raw.get("candidates")
+ snippets = {row["path"]: row["snippet"] for row in served["results"]}
+ assert snippets == {
+ "src/session/cache.py": "Keeps session rows warm.",
+ "pkg/store.go": "Session store backed by a map.",
+ "lib/layer.rb": "Cache layer with no heading. ## not a heading here",
+ "docs/only.md": "# docs/only.md\n\n## Overview\n",
+ }