Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(search): let a page be too thin to be worth indexing (#1198)

Swati Ahuja81c74625cb90
diff --git a/packages/core/src/repowise/core/persistence/information_floor.py b/packages/core/src/repowise/core/persistence/information_floor.py
new file mode 100644
index 000000000..976b1e357
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/information_floor.py
@@ -0,0 +1,144 @@
+"""How much a page actually says, and whether that is enough to index it.
+
+Search fetches a fixed number of rows before it filters anything, so every
+indexed page occupies a slot whether or not it can answer a question. A page
+that restates its own filename and then says three sections' worth of
+nothing still takes one of those slots from a page that could have.
+
+The page itself is kept — it is a valid link target and a reader arriving at
+it learns the file exists and has no callers. It is only kept out of the
+index, where its cost is paid by other pages.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+__all__ = [
+ "DEFAULT_INFORMATION_FLOOR",
+ "INFORMATION_FLOOR_ENV",
+ "information_floor",
+ "meets_information_floor",
+ "substantive_text",
+]
+
+# Chars of substantive prose a page needs before it is worth a search slot.
+#
+# Zero: off. Nothing is excluded until a value is set, so installing this
+# changes no one's index.
+#
+# The mechanism ships ahead of the number on purpose. Excluding pages is a
+# deletion, and what it costs is a retrieval measurement rather than an
+# argument — a thin page can still be the only route to a file, in which case
+# holding it out loses the file. Measured against a 3,678-page corpus:
+#
+# floor 200 → 146 pages excluded (4.0%)
+# floor 300 → 516 (14.0%)
+# floor 400 → 914 (24.9%)
+# floor 500 → 1186 (32.2%)
+#
+# No orientation page is excluded at any of those levels — no layer page, no
+# onboarding page, no architecture diagram. What goes is file pages whose
+# every section is a placeholder, and symbol spotlights that restate a
+# signature. 300 is the level at which what is dropped is plainly not an
+# answer to anything, and is the first value worth sweeping.
+DEFAULT_INFORMATION_FLOOR = 0
+
+# Tunable without a redeploy, because the right value depends on how verbose a
+# repository's generated pages are, and finding it means sweeping it.
+INFORMATION_FLOOR_ENV = "REPOWISE_INDEX_INFORMATION_FLOOR"
+
+_FENCE = re.compile(r"^\s*```")
+_HEADING = re.compile(r"^\s*#{1,6}\s")
+# ``_No internal dependencies resolved._`` and its siblings: a whole line in
+# italics, which in these templates is only ever a "there is nothing here".
+_PLACEHOLDER = re.compile(r"^\s*_[^_]*_\s*$")
+# ``**Kind:** interface | **Defined in:** ...`` — a metadata strip, not prose.
+_FIELD_LINE = re.compile(r"^\s*\*\*[^*]+:\*\*")
+_TABLE_ROW = re.compile(r"^\s*\|")
+_HORIZONTAL_RULE = re.compile(r"^\s*-{3,}\s*$")
+# The provenance trailer every generated page ends with, in italics. It is on
+# 97% of pages in the measured corpus and says the same thing on all of them.
+#
+# Three things open with ``*`` and only one of them is this: a bullet written
+# ``* item`` (a space follows) and a bold field line ``**Kind:** …`` (another
+# star follows). Both are excluded here, and the field line is also classified
+# before this runs — matching it as an unterminated italic opened a block that
+# never closed and swallowed every page that carried one.
+_ITALIC_OPEN = re.compile(r"^\s*\*(?![\s*])")
+_ITALIC_CLOSE = re.compile(r"\*\s*$")
+
+
+def substantive_text(content: str) -> str:
+ """The prose of *content*, with what every page carries stripped out.
+
+ Removed: fenced code, headings, placeholder lines, metadata field lines,
+ table rows, horizontal rules and the italic provenance trailer.
+
+ Bullet lists are *kept*. They look like structure and are not: on a file
+ page the dependency and caller lists are the page's most specific
+ content, and on a layer or onboarding page the bullets are the page.
+ Dropping them takes the exclusion from 3% of the measured corpus to 36%
+ and starts excluding orientation pages, which are the ones a reader most
+ needs to find.
+ """
+ out: list[str] = []
+ in_fence = False
+ in_italics = False
+ for line in content.splitlines():
+ if _FENCE.match(line):
+ in_fence = not in_fence
+ continue
+ if in_fence or not line.strip():
+ continue
+ # The trailer wraps over several lines and only the first carries the
+ # opening marker, so it is consumed as a block rather than per line.
+ if in_italics:
+ if _ITALIC_CLOSE.search(line):
+ in_italics = False
+ continue
+ # Classified before emphasis, so a line that merely starts with a star
+ # cannot be mistaken for an italic block that never ends.
+ if (
+ _HEADING.match(line)
+ or _PLACEHOLDER.match(line)
+ or _FIELD_LINE.match(line)
+ or _TABLE_ROW.match(line)
+ or _HORIZONTAL_RULE.match(line)
+ ):
+ continue
+ if _ITALIC_OPEN.match(line):
+ if not _ITALIC_CLOSE.search(line.strip()[1:]):
+ in_italics = True
+ continue
+ out.append(line.strip())
+ return " ".join(out)
+
+
+def information_floor() -> int:
+ """The active floor, from the environment or the default.
+
+ An unparseable or negative value falls back to the default rather than
+ disabling the floor: a typo in a deployment variable should not silently
+ change what is in the index.
+ """
+ raw = os.environ.get(INFORMATION_FLOOR_ENV)
+ if raw is None:
+ return DEFAULT_INFORMATION_FLOOR
+ try:
+ value = int(raw)
+ except ValueError:
+ return DEFAULT_INFORMATION_FLOOR
+ return value if value >= 0 else DEFAULT_INFORMATION_FLOOR
+
+
+def meets_information_floor(content: str, floor: int | None = None) -> bool:
+ """Whether *content* says enough to be worth a slot in the index.
+
+ A floor of 0 admits everything, which is how the feature is turned off.
+ """
+ limit = information_floor() if floor is None else floor
+ if limit <= 0:
+ return True
+ return len(substantive_text(content)) >= limit
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index dde84117e..a446a68be 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -26,6 +26,8 @@ from typing import Literal
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.sql import text
+from .information_floor import information_floor, meets_information_floor, substantive_text
+
@dataclass
class SearchResult:
@@ -308,6 +310,11 @@ class FullTextSearch:
# term → how many pages it matches. "" is the corpus size.
self._df_cache: OrderedDict[str, int] = OrderedDict()
self._warned_missing_fields = False
+ # Pages this instance kept out of the index for saying too little.
+ # A counter, not a line per page: a corpus-wide write would print
+ # thousands. The total is what says whether the floor is set sanely.
+ self.skipped_below_floor = 0
+ self._floor_sample: list[str] = []
async def ensure_index(self) -> None:
"""Create the FTS index if it does not exist (idempotent).
@@ -420,19 +427,35 @@ class FullTextSearch:
An empty *string* is a different thing and is not warned about: some
page types genuinely have no summary.
+
+ A page below the information floor is not indexed, and any row it
+ already had is deleted. Ten places write this index, so the decision
+ lives here rather than at each of them — one that skipped the check
+ would keep re-admitting the pages the others exclude, and nothing
+ would report the disagreement. The page itself is untouched: it stays
+ in ``wiki_pages`` and stays a valid link target.
"""
if summary is None or target_path is None:
self._warn_missing_index_fields(page_id, summary, target_path)
summary = summary if summary is not None else ""
target_path = target_path if target_path is not None else ""
+ indexable = meets_information_floor(content)
+ if not indexable:
+ self._count_skipped_below_floor(page_id, content)
+
if self._dialect == "sqlite":
async with self._engine.begin() as conn:
- # FTS5 does not support UPDATE; use DELETE + INSERT
+ # FTS5 does not support UPDATE; use DELETE + INSERT. The delete
+ # runs either way: a page that has fallen below the floor since
+ # its last index must lose the row it had, or the exclusion
+ # only ever applies to pages that never existed.
await conn.execute(
text("DELETE FROM page_fts WHERE page_id = :pid"),
{"pid": page_id},
)
+ if not indexable:
+ return
await conn.execute(
text(
"INSERT INTO page_fts(page_id, title, content, summary, target_path) "
@@ -449,6 +472,34 @@ class FullTextSearch:
# PostgreSQL: the GIN index on wiki_pages is maintained automatically
# by the database as rows are inserted/updated via the CRUD layer.
+ def _count_skipped_below_floor(self, page_id: str, content: str) -> None:
+ """Record a page held out of the index, and name the first few."""
+ self.skipped_below_floor += 1
+ if len(self._floor_sample) < 3:
+ self._floor_sample.append(page_id)
+ if self.skipped_below_floor == 1:
+ _log.info(
+ "fts_page_below_information_floor page_id=%s substantive_chars=%d floor=%d",
+ page_id,
+ len(substantive_text(content)),
+ information_floor(),
+ )
+
+ def log_floor_exclusions(self) -> None:
+ """Report the run's total. Callers that index a corpus call this once.
+
+ Silent when nothing was excluded, so a repository whose pages are all
+ substantial says nothing rather than reporting a zero.
+ """
+ if not self.skipped_below_floor:
+ return
+ _log.info(
+ "fts_pages_below_information_floor count=%d floor=%d sample=%s",
+ self.skipped_below_floor,
+ information_floor(),
+ ", ".join(self._floor_sample),
+ )
+
def _warn_missing_index_fields(
self, page_id: str, summary: str | None, target_path: str | None
) -> None:
diff --git a/tests/unit/persistence/test_information_floor.py b/tests/unit/persistence/test_information_floor.py
new file mode 100644
index 000000000..6e9fffa45
--- /dev/null
+++ b/tests/unit/persistence/test_information_floor.py
@@ -0,0 +1,287 @@
+"""Pages that say almost nothing stay out of the search index.
+
+Search fetches a fixed number of rows before it filters anything, so every
+indexed page holds a slot whether or not it can answer a question. A file
+page whose every section is a placeholder holds one of those slots against a
+page that could have used it.
+
+The page is kept. Only its index row goes.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+from repowise.core.persistence.information_floor import (
+ DEFAULT_INFORMATION_FLOOR,
+ INFORMATION_FLOOR_ENV,
+ # 300 is the first value worth sweeping; the tests that exercise the floor
+ # set it explicitly, because shipping it on would change every index.
+ information_floor,
+ meets_information_floor,
+ substantive_text,
+)
+
+# A real file page from the measured corpus, at 186 substantive characters:
+# every section below the Overview is a placeholder.
+SKELETON = """# packages/vscode/esbuild.mjs
+
+## Overview
+
+`packages/vscode/esbuild.mjs` is a javascript source file in the Application layer.
+
+## Public API
+
+_No public symbols were extracted from this file._
+
+## Depends on
+
+_No internal dependencies resolved._
+
+## Used by
+
+_No internal callers were resolved for this file._
+
+## Usage Notes
+
+**Layer:** Application | **Role:** entry_point
+
+---
+
+*Built from the code itself: parsed symbols, the import graph, git history and
+the knowledge graph. Every statement here is checked against the source rather
+than written about it.*
+"""
+
+SWEEP_FLOOR = 300
+
+
[email protected](autouse=True)
+def floor_on(monkeypatch):
+ """The floor ships off. These tests are about what it does when on."""
+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, str(SWEEP_FLOOR))
+
+
+SUBSTANTIAL = (
+ "# packages/core/search.py\n\n## Overview\n\n"
+ + "The full-text index is an FTS5 virtual table rebuilt on demand. " * 12
+)
+
+
+# --------------------------------------------------------------------------
+# What counts as substance
+# --------------------------------------------------------------------------
+
+
+def test_a_heading_is_not_content():
+ assert substantive_text("## Overview\n### Details") == ""
+
+
+def test_a_placeholder_line_is_not_content():
+ assert substantive_text("_No internal dependencies resolved._") == ""
+
+
+def test_a_metadata_field_line_is_not_content():
+ assert substantive_text("**Kind:** interface | **Defined in:** `a/b.ts`") == ""
+
+
+def test_a_table_is_not_content():
+ assert substantive_text("| Symbol | Kind |\n| --- | --- |\n| `config` | variable |") == ""
+
+
+def test_fenced_code_is_not_content():
+ """The signature block is the file restating itself, not prose about it."""
+ assert substantive_text("```\ninterface GraphLike\n```") == ""
+
+
+def test_the_provenance_trailer_is_not_content():
+ """It is on 97% of pages in the measured corpus and says the same thing.
+
+ Left in, it adds ~180 characters to every page and no floor discriminates.
+ """
+ trailer = "*Built from the code itself: parsed symbols, the import graph.*"
+ assert substantive_text(trailer) == ""
+
+
+def test_bullet_lists_are_content():
+ """They look like structure and are not.
+
+ On a file page the dependency and caller lists are the most specific thing
+ it says, and on a layer or onboarding page the bullets are the page.
+ Dropping them takes the exclusion from 3% of the measured corpus to 36%
+ and starts excluding the orientation pages a reader most needs to find.
+ """
+ text = substantive_text("- `packages/types/src/graph.ts`\n- `packages/ui/src/panel.tsx`")
+ assert "packages/types/src/graph.ts" in text
+ assert "packages/ui/src/panel.tsx" in text
+
+
+def test_prose_survives_stripping_line_by_line():
+ assert substantive_text(SKELETON) == (
+ "`packages/vscode/esbuild.mjs` is a javascript source file in the Application layer."
+ )
+
+
+# --------------------------------------------------------------------------
+# The floor itself
+# --------------------------------------------------------------------------
+
+
+def test_a_skeleton_page_is_below_the_floor():
+ assert meets_information_floor(SKELETON) is False
+
+
+def test_a_real_page_is_above_the_floor():
+ assert meets_information_floor(SUBSTANTIAL) is True
+
+
+def test_the_floor_reads_the_environment(monkeypatch):
+ """Tunable without a redeploy: the right value is a property of a corpus."""
+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "50")
+ assert information_floor() == 50
+ assert meets_information_floor(SKELETON) is True
+
+
+def test_a_floor_of_zero_admits_everything(monkeypatch):
+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "0")
+ assert meets_information_floor("") is True
+
+
+def test_an_unparseable_floor_falls_back_to_the_default(monkeypatch):
+ """A typo in a deployment variable must not silently change the index."""
+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "three hundred")
+ assert information_floor() == DEFAULT_INFORMATION_FLOOR
+
+
+def test_a_negative_floor_falls_back_to_the_default(monkeypatch):
+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "-1")
+ assert information_floor() == DEFAULT_INFORMATION_FLOOR
+
+
+# --------------------------------------------------------------------------
+# What it does to the index
+# --------------------------------------------------------------------------
+
+
+async def fts(async_engine):
+ from repowise.core.persistence import FullTextSearch
+
+ index = FullTextSearch(async_engine)
+ await index.ensure_index()
+ return index
+
+
+async def test_a_thin_page_is_not_searchable(fts):
+ await fts.index(
+ "file_page:esbuild.mjs",
+ "File: esbuild.mjs",
+ SKELETON,
+ summary="",
+ target_path="packages/vscode/esbuild.mjs",
+ )
+
+ assert await fts.search("javascript source file") == []
+
+
+async def test_a_substantial_page_still_is(fts):
+ await fts.index(
+ "file_page:search.py",
+ "File: search.py",
+ SUBSTANTIAL,
+ summary="",
+ target_path="packages/core/search.py",
+ )
+
+ assert [r.page_id for r in await fts.search("FTS5 virtual table")] == ["file_page:search.py"]
+
+
+async def test_a_page_that_falls_below_the_floor_loses_the_row_it_had(fts):
+ """Otherwise the exclusion only ever applies to pages that never existed.
+
+ A page regenerated thinner than it was keeps serving its old text, which
+ is worse than the skeleton it became: the index and the page disagree.
+ """
+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")
+ assert await fts.search("FTS5 virtual table") != []
+
+ await fts.index("file_page:a.py", "File: a.py", SKELETON, summary="", target_path="a.py")
+
+ assert await fts.search("FTS5 virtual table") == []
+ assert await fts.search("javascript source file") == []
+
+
+async def test_the_run_counts_what_it_held_out(fts, caplog):
+ """A counter, not a line per page — a corpus write would print thousands."""
+ with caplog.at_level(logging.INFO, logger="repowise.core.persistence.search"):
+ for i in range(4):
+ await fts.index(
+ f"file_page:{i}.py", f"File: {i}.py", SKELETON, summary="", target_path=f"{i}.py"
+ )
+ fts.log_floor_exclusions()
+
+ assert fts.skipped_below_floor == 4
+
+ totals = [
+ r.getMessage()
+ for r in caplog.records
+ if "fts_pages_below_information_floor" in r.getMessage()
+ ]
+ assert len(totals) == 1
+ assert "count=4" in totals[0]
+ assert f"floor={SWEEP_FLOOR}" in totals[0]
+
+
+async def test_nothing_excluded_reports_nothing(fts, caplog):
+ """A repository whose pages are all substantial says nothing, not zero."""
+ with caplog.at_level(logging.INFO, logger="repowise.core.persistence.search"):
+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")
+ fts.log_floor_exclusions()
+
+ assert fts.skipped_below_floor == 0
+ assert [r for r in caplog.records if "below_information_floor" in r.getMessage()] == []
+
+
+def test_a_star_bullet_is_content_not_emphasis():
+ """``* item`` and ``*emphasis*`` open with the same character.
+
+ The one with a space after it is a list; the one without is the trailer.
+ """
+ assert "keep me" in substantive_text("* keep me")
+ assert substantive_text("*drop me*") == ""
+
+
+def test_a_multi_line_italic_block_is_consumed_whole():
+ """The trailer wraps, and only its first line carries the open marker."""
+ assert substantive_text("*first line\nsecond line\nthird line*") == ""
+ assert substantive_text("*trailer\nwraps here*\n\nreal prose") == "real prose"
+
+
+def test_a_bold_field_line_does_not_open_an_italic_block():
+ """``**Kind:**`` starts with a star and never closes one.
+
+ Read as an unterminated italic it opened a block that consumed every
+ remaining line, so a page carrying one metadata strip measured as saying
+ nothing at all — which excluded every spotlight and every API contract in
+ the corpus this was measured against.
+ """
+ page = "**Kind:** class | **Defined in:** `a/b.py`\n\n## Overview\n\nReal prose here."
+
+ assert substantive_text(page) == "Real prose here."
+
+
+def test_the_floor_is_off_unless_it_is_set(monkeypatch):
+ """Installing this must not change anyone's index.
+
+ Excluding pages is a deletion, and what it costs is a retrieval
+ measurement rather than an argument. The mechanism ships ahead of the
+ number so the number can be swept.
+ """
+ monkeypatch.delenv(INFORMATION_FLOOR_ENV, raising=False)
+
+ assert DEFAULT_INFORMATION_FLOOR == 0
+ assert information_floor() == 0
+ assert meets_information_floor(SKELETON) is True
+ assert meets_information_floor("") is True