Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(search): let a page be too thin to be worth indexing (#1198)
Swati Ahuja81c74625cb90
diff --git a/packages/core/src/repowise/core/persistence/information_floor.py b/packages/core/src/repowise/core/persistence/information_floor.pynew file mode 100644index 000000000..976b1e357--- /dev/null+++ b/packages/core/src/repowise/core/persistence/information_floor.py@@ -0,0 +1,144 @@+"""How much a page actually says, and whether that is enough to index it.++Search fetches a fixed number of rows before it filters anything, so every+indexed page occupies a slot whether or not it can answer a question. A page+that restates its own filename and then says three sections' worth of+nothing still takes one of those slots from a page that could have.++The page itself is kept — it is a valid link target and a reader arriving at+it learns the file exists and has no callers. It is only kept out of the+index, where its cost is paid by other pages.+"""++from __future__ import annotations++import os+import re++__all__ = [+ "DEFAULT_INFORMATION_FLOOR",+ "INFORMATION_FLOOR_ENV",+ "information_floor",+ "meets_information_floor",+ "substantive_text",+]++# Chars of substantive prose a page needs before it is worth a search slot.+#+# Zero: off. Nothing is excluded until a value is set, so installing this+# changes no one's index.+#+# The mechanism ships ahead of the number on purpose. Excluding pages is a+# deletion, and what it costs is a retrieval measurement rather than an+# argument — a thin page can still be the only route to a file, in which case+# holding it out loses the file. Measured against a 3,678-page corpus:+#+# floor 200 → 146 pages excluded (4.0%)+# floor 300 → 516 (14.0%)+# floor 400 → 914 (24.9%)+# floor 500 → 1186 (32.2%)+#+# No orientation page is excluded at any of those levels — no layer page, no+# onboarding page, no architecture diagram. What goes is file pages whose+# every section is a placeholder, and symbol spotlights that restate a+# signature. 300 is the level at which what is dropped is plainly not an+# answer to anything, and is the first value worth sweeping.+DEFAULT_INFORMATION_FLOOR = 0++# Tunable without a redeploy, because the right value depends on how verbose a+# repository's generated pages are, and finding it means sweeping it.+INFORMATION_FLOOR_ENV = "REPOWISE_INDEX_INFORMATION_FLOOR"++_FENCE = re.compile(r"^\s*```")+_HEADING = re.compile(r"^\s*#{1,6}\s")+# ``_No internal dependencies resolved._`` and its siblings: a whole line in+# italics, which in these templates is only ever a "there is nothing here".+_PLACEHOLDER = re.compile(r"^\s*_[^_]*_\s*$")+# ``**Kind:** interface | **Defined in:** ...`` — a metadata strip, not prose.+_FIELD_LINE = re.compile(r"^\s*\*\*[^*]+:\*\*")+_TABLE_ROW = re.compile(r"^\s*\|")+_HORIZONTAL_RULE = re.compile(r"^\s*-{3,}\s*$")+# The provenance trailer every generated page ends with, in italics. It is on+# 97% of pages in the measured corpus and says the same thing on all of them.+#+# Three things open with ``*`` and only one of them is this: a bullet written+# ``* item`` (a space follows) and a bold field line ``**Kind:** …`` (another+# star follows). Both are excluded here, and the field line is also classified+# before this runs — matching it as an unterminated italic opened a block that+# never closed and swallowed every page that carried one.+_ITALIC_OPEN = re.compile(r"^\s*\*(?![\s*])")+_ITALIC_CLOSE = re.compile(r"\*\s*$")+++def substantive_text(content: str) -> str:+ """The prose of *content*, with what every page carries stripped out.++ Removed: fenced code, headings, placeholder lines, metadata field lines,+ table rows, horizontal rules and the italic provenance trailer.++ Bullet lists are *kept*. They look like structure and are not: on a file+ page the dependency and caller lists are the page's most specific+ content, and on a layer or onboarding page the bullets are the page.+ Dropping them takes the exclusion from 3% of the measured corpus to 36%+ and starts excluding orientation pages, which are the ones a reader most+ needs to find.+ """+ out: list[str] = []+ in_fence = False+ in_italics = False+ for line in content.splitlines():+ if _FENCE.match(line):+ in_fence = not in_fence+ continue+ if in_fence or not line.strip():+ continue+ # The trailer wraps over several lines and only the first carries the+ # opening marker, so it is consumed as a block rather than per line.+ if in_italics:+ if _ITALIC_CLOSE.search(line):+ in_italics = False+ continue+ # Classified before emphasis, so a line that merely starts with a star+ # cannot be mistaken for an italic block that never ends.+ if (+ _HEADING.match(line)+ or _PLACEHOLDER.match(line)+ or _FIELD_LINE.match(line)+ or _TABLE_ROW.match(line)+ or _HORIZONTAL_RULE.match(line)+ ):+ continue+ if _ITALIC_OPEN.match(line):+ if not _ITALIC_CLOSE.search(line.strip()[1:]):+ in_italics = True+ continue+ out.append(line.strip())+ return " ".join(out)+++def information_floor() -> int:+ """The active floor, from the environment or the default.++ An unparseable or negative value falls back to the default rather than+ disabling the floor: a typo in a deployment variable should not silently+ change what is in the index.+ """+ raw = os.environ.get(INFORMATION_FLOOR_ENV)+ if raw is None:+ return DEFAULT_INFORMATION_FLOOR+ try:+ value = int(raw)+ except ValueError:+ return DEFAULT_INFORMATION_FLOOR+ return value if value >= 0 else DEFAULT_INFORMATION_FLOOR+++def meets_information_floor(content: str, floor: int | None = None) -> bool:+ """Whether *content* says enough to be worth a slot in the index.++ A floor of 0 admits everything, which is how the feature is turned off.+ """+ limit = information_floor() if floor is None else floor+ if limit <= 0:+ return True+ return len(substantive_text(content)) >= limitdiff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex dde84117e..a446a68be 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -26,6 +26,8 @@ from typing import Literalfrom sqlalchemy.ext.asyncio import AsyncEnginefrom sqlalchemy.sql import text+from .information_floor import information_floor, meets_information_floor, substantive_text+@dataclassclass SearchResult:@@ -308,6 +310,11 @@ class FullTextSearch:# term → how many pages it matches. "" is the corpus size.self._df_cache: OrderedDict[str, int] = OrderedDict()self._warned_missing_fields = False+ # Pages this instance kept out of the index for saying too little.+ # A counter, not a line per page: a corpus-wide write would print+ # thousands. The total is what says whether the floor is set sanely.+ self.skipped_below_floor = 0+ self._floor_sample: list[str] = []async def ensure_index(self) -> None:"""Create the FTS index if it does not exist (idempotent).@@ -420,19 +427,35 @@ class FullTextSearch:An empty *string* is a different thing and is not warned about: somepage types genuinely have no summary.++ A page below the information floor is not indexed, and any row it+ already had is deleted. Ten places write this index, so the decision+ lives here rather than at each of them — one that skipped the check+ would keep re-admitting the pages the others exclude, and nothing+ would report the disagreement. The page itself is untouched: it stays+ in ``wiki_pages`` and stays a valid link target."""if summary is None or target_path is None:self._warn_missing_index_fields(page_id, summary, target_path)summary = summary if summary is not None else ""target_path = target_path if target_path is not None else ""+ indexable = meets_information_floor(content)+ if not indexable:+ self._count_skipped_below_floor(page_id, content)+if self._dialect == "sqlite":async with self._engine.begin() as conn:- # FTS5 does not support UPDATE; use DELETE + INSERT+ # FTS5 does not support UPDATE; use DELETE + INSERT. The delete+ # runs either way: a page that has fallen below the floor since+ # its last index must lose the row it had, or the exclusion+ # only ever applies to pages that never existed.await conn.execute(text("DELETE FROM page_fts WHERE page_id = :pid"),{"pid": page_id},)+ if not indexable:+ returnawait conn.execute(text("INSERT INTO page_fts(page_id, title, content, summary, target_path) "@@ -449,6 +472,34 @@ class FullTextSearch:# PostgreSQL: the GIN index on wiki_pages is maintained automatically# by the database as rows are inserted/updated via the CRUD layer.+ def _count_skipped_below_floor(self, page_id: str, content: str) -> None:+ """Record a page held out of the index, and name the first few."""+ self.skipped_below_floor += 1+ if len(self._floor_sample) < 3:+ self._floor_sample.append(page_id)+ if self.skipped_below_floor == 1:+ _log.info(+ "fts_page_below_information_floor page_id=%s substantive_chars=%d floor=%d",+ page_id,+ len(substantive_text(content)),+ information_floor(),+ )++ def log_floor_exclusions(self) -> None:+ """Report the run's total. Callers that index a corpus call this once.++ Silent when nothing was excluded, so a repository whose pages are all+ substantial says nothing rather than reporting a zero.+ """+ if not self.skipped_below_floor:+ return+ _log.info(+ "fts_pages_below_information_floor count=%d floor=%d sample=%s",+ self.skipped_below_floor,+ information_floor(),+ ", ".join(self._floor_sample),+ )+def _warn_missing_index_fields(self, page_id: str, summary: str | None, target_path: str | None) -> None:diff --git a/tests/unit/persistence/test_information_floor.py b/tests/unit/persistence/test_information_floor.pynew file mode 100644index 000000000..6e9fffa45--- /dev/null+++ b/tests/unit/persistence/test_information_floor.py@@ -0,0 +1,287 @@+"""Pages that say almost nothing stay out of the search index.++Search fetches a fixed number of rows before it filters anything, so every+indexed page holds a slot whether or not it can answer a question. A file+page whose every section is a placeholder holds one of those slots against a+page that could have used it.++The page is kept. Only its index row goes.+"""++from __future__ import annotations++import logging++import pytest++from repowise.core.persistence.information_floor import (+ DEFAULT_INFORMATION_FLOOR,+ INFORMATION_FLOOR_ENV,+ # 300 is the first value worth sweeping; the tests that exercise the floor+ # set it explicitly, because shipping it on would change every index.+ information_floor,+ meets_information_floor,+ substantive_text,+)++# A real file page from the measured corpus, at 186 substantive characters:+# every section below the Overview is a placeholder.+SKELETON = """# packages/vscode/esbuild.mjs++## Overview++`packages/vscode/esbuild.mjs` is a javascript source file in the Application layer.++## Public API++_No public symbols were extracted from this file._++## Depends on++_No internal dependencies resolved._++## Used by++_No internal callers were resolved for this file._++## Usage Notes++**Layer:** Application | **Role:** entry_point++---++*Built from the code itself: parsed symbols, the import graph, git history and+the knowledge graph. Every statement here is checked against the source rather+than written about it.*+"""++SWEEP_FLOOR = 300++[email protected](autouse=True)+def floor_on(monkeypatch):+ """The floor ships off. These tests are about what it does when on."""+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, str(SWEEP_FLOOR))+++SUBSTANTIAL = (+ "# packages/core/search.py\n\n## Overview\n\n"+ + "The full-text index is an FTS5 virtual table rebuilt on demand. " * 12+)+++# --------------------------------------------------------------------------+# What counts as substance+# --------------------------------------------------------------------------+++def test_a_heading_is_not_content():+ assert substantive_text("## Overview\n### Details") == ""+++def test_a_placeholder_line_is_not_content():+ assert substantive_text("_No internal dependencies resolved._") == ""+++def test_a_metadata_field_line_is_not_content():+ assert substantive_text("**Kind:** interface | **Defined in:** `a/b.ts`") == ""+++def test_a_table_is_not_content():+ assert substantive_text("| Symbol | Kind |\n| --- | --- |\n| `config` | variable |") == ""+++def test_fenced_code_is_not_content():+ """The signature block is the file restating itself, not prose about it."""+ assert substantive_text("```\ninterface GraphLike\n```") == ""+++def test_the_provenance_trailer_is_not_content():+ """It is on 97% of pages in the measured corpus and says the same thing.++ Left in, it adds ~180 characters to every page and no floor discriminates.+ """+ trailer = "*Built from the code itself: parsed symbols, the import graph.*"+ assert substantive_text(trailer) == ""+++def test_bullet_lists_are_content():+ """They look like structure and are not.++ On a file page the dependency and caller lists are the most specific thing+ it says, and on a layer or onboarding page the bullets are the page.+ Dropping them takes the exclusion from 3% of the measured corpus to 36%+ and starts excluding the orientation pages a reader most needs to find.+ """+ text = substantive_text("- `packages/types/src/graph.ts`\n- `packages/ui/src/panel.tsx`")+ assert "packages/types/src/graph.ts" in text+ assert "packages/ui/src/panel.tsx" in text+++def test_prose_survives_stripping_line_by_line():+ assert substantive_text(SKELETON) == (+ "`packages/vscode/esbuild.mjs` is a javascript source file in the Application layer."+ )+++# --------------------------------------------------------------------------+# The floor itself+# --------------------------------------------------------------------------+++def test_a_skeleton_page_is_below_the_floor():+ assert meets_information_floor(SKELETON) is False+++def test_a_real_page_is_above_the_floor():+ assert meets_information_floor(SUBSTANTIAL) is True+++def test_the_floor_reads_the_environment(monkeypatch):+ """Tunable without a redeploy: the right value is a property of a corpus."""+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "50")+ assert information_floor() == 50+ assert meets_information_floor(SKELETON) is True+++def test_a_floor_of_zero_admits_everything(monkeypatch):+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "0")+ assert meets_information_floor("") is True+++def test_an_unparseable_floor_falls_back_to_the_default(monkeypatch):+ """A typo in a deployment variable must not silently change the index."""+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "three hundred")+ assert information_floor() == DEFAULT_INFORMATION_FLOOR+++def test_a_negative_floor_falls_back_to_the_default(monkeypatch):+ monkeypatch.setenv(INFORMATION_FLOOR_ENV, "-1")+ assert information_floor() == DEFAULT_INFORMATION_FLOOR+++# --------------------------------------------------------------------------+# What it does to the index+# --------------------------------------------------------------------------+++async def fts(async_engine):+ from repowise.core.persistence import FullTextSearch++ index = FullTextSearch(async_engine)+ await index.ensure_index()+ return index+++async def test_a_thin_page_is_not_searchable(fts):+ await fts.index(+ "file_page:esbuild.mjs",+ "File: esbuild.mjs",+ SKELETON,+ summary="",+ target_path="packages/vscode/esbuild.mjs",+ )++ assert await fts.search("javascript source file") == []+++async def test_a_substantial_page_still_is(fts):+ await fts.index(+ "file_page:search.py",+ "File: search.py",+ SUBSTANTIAL,+ summary="",+ target_path="packages/core/search.py",+ )++ assert [r.page_id for r in await fts.search("FTS5 virtual table")] == ["file_page:search.py"]+++async def test_a_page_that_falls_below_the_floor_loses_the_row_it_had(fts):+ """Otherwise the exclusion only ever applies to pages that never existed.++ A page regenerated thinner than it was keeps serving its old text, which+ is worse than the skeleton it became: the index and the page disagree.+ """+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")+ assert await fts.search("FTS5 virtual table") != []++ await fts.index("file_page:a.py", "File: a.py", SKELETON, summary="", target_path="a.py")++ assert await fts.search("FTS5 virtual table") == []+ assert await fts.search("javascript source file") == []+++async def test_the_run_counts_what_it_held_out(fts, caplog):+ """A counter, not a line per page — a corpus write would print thousands."""+ with caplog.at_level(logging.INFO, logger="repowise.core.persistence.search"):+ for i in range(4):+ await fts.index(+ f"file_page:{i}.py", f"File: {i}.py", SKELETON, summary="", target_path=f"{i}.py"+ )+ fts.log_floor_exclusions()++ assert fts.skipped_below_floor == 4++ totals = [+ r.getMessage()+ for r in caplog.records+ if "fts_pages_below_information_floor" in r.getMessage()+ ]+ assert len(totals) == 1+ assert "count=4" in totals[0]+ assert f"floor={SWEEP_FLOOR}" in totals[0]+++async def test_nothing_excluded_reports_nothing(fts, caplog):+ """A repository whose pages are all substantial says nothing, not zero."""+ with caplog.at_level(logging.INFO, logger="repowise.core.persistence.search"):+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")+ fts.log_floor_exclusions()++ assert fts.skipped_below_floor == 0+ assert [r for r in caplog.records if "below_information_floor" in r.getMessage()] == []+++def test_a_star_bullet_is_content_not_emphasis():+ """``* item`` and ``*emphasis*`` open with the same character.++ The one with a space after it is a list; the one without is the trailer.+ """+ assert "keep me" in substantive_text("* keep me")+ assert substantive_text("*drop me*") == ""+++def test_a_multi_line_italic_block_is_consumed_whole():+ """The trailer wraps, and only its first line carries the open marker."""+ assert substantive_text("*first line\nsecond line\nthird line*") == ""+ assert substantive_text("*trailer\nwraps here*\n\nreal prose") == "real prose"+++def test_a_bold_field_line_does_not_open_an_italic_block():+ """``**Kind:**`` starts with a star and never closes one.++ Read as an unterminated italic it opened a block that consumed every+ remaining line, so a page carrying one metadata strip measured as saying+ nothing at all — which excluded every spotlight and every API contract in+ the corpus this was measured against.+ """+ page = "**Kind:** class | **Defined in:** `a/b.py`\n\n## Overview\n\nReal prose here."++ assert substantive_text(page) == "Real prose here."+++def test_the_floor_is_off_unless_it_is_set(monkeypatch):+ """Installing this must not change anyone's index.++ Excluding pages is a deletion, and what it costs is a retrieval+ measurement rather than an argument. The mechanism ships ahead of the+ number so the number can be swept.+ """+ monkeypatch.delenv(INFORMATION_FLOOR_ENV, raising=False)++ assert DEFAULT_INFORMATION_FLOOR == 0+ assert information_floor() == 0+ assert meets_information_floor(SKELETON) is True+ assert meets_information_floor("") is True