Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(update): retire template pages the selection no longer picks (#4211)

Raghav Chamadiyabdebde6b5dc2
diff --git a/packages/core/alembic/versions/0116_page_versions_page_id_index.py b/packages/core/alembic/versions/0116_page_versions_page_id_index.py
new file mode 100644
index 000000000..4b94d121e
--- /dev/null
+++ b/packages/core/alembic/versions/0116_page_versions_page_id_index.py
@@ -0,0 +1,28 @@
+"""Index wiki_page_versions.page_id, the foreign key every page delete checks.
+
+Local SQLite gets the index from ``init_db``.
+
+Revision ID: 0116
+Revises: 0115
+Create Date: 2026-10-19
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers
+revision: str = "0116"
+down_revision: str | None = "0115"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ op.create_index("ix_wiki_page_versions_page_id", "wiki_page_versions", ["page_id"])
+
+
+def downgrade() -> None:
+ op.drop_index("ix_wiki_page_versions_page_id", table_name="wiki_page_versions")
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index d58bd50a7..462080e24 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -283,6 +283,28 @@ def count_documentable_files(parsed_files: list[Any]) -> int:
return sum(1 for p in parsed_files if _is_code_file(p) and _passes_importance_floor(p))
+def file_page_eligible(parsed: Any, *, is_hotspot: bool) -> bool:
+ """Whether *parsed* clears every per-file gate of file-page selection.
+
+ Repo-wide gates (cap, near-clone dedupe) are not asked: an update rendering
+ the file alone would still give it a page.
+ """
+ return (
+ _is_code_file(parsed)
+ and _passes_importance_floor(parsed)
+ # Graph metrics only add to a positive score.
+ and score_file(
+ parsed,
+ pagerank=0.0,
+ betweenness=0.0,
+ max_pagerank=0.0,
+ max_betweenness=0.0,
+ is_hotspot=is_hotspot,
+ )
+ > 0.0
+ )
+
+
def _passes_importance_floor(parsed: Any) -> bool:
"""Whether *parsed* is worth a file page at all.
@@ -365,6 +387,14 @@ def _build_symbol_candidates(
inputs: SelectionInputs,
) -> list[tuple[float, tuple[str, str]]]:
"""Return ``[(score, (file_path, symbol_name)), ...]`` descending."""
+ ranked, keep = rank_symbol_candidates(inputs)
+ return ranked[:keep]
+
+
+def rank_symbol_candidates(
+ inputs: SelectionInputs,
+) -> tuple[list[tuple[float, tuple[str, str]]], int]:
+ """Every production spotlight candidate in rank order, and how many selection keeps."""
max_pr = max(inputs.pagerank.values(), default=0.0)
test_paths = {p.file_info.path for p in inputs.parsed_files if p.file_info.is_test}
scored: list[tuple[float, tuple[str, str]]] = []
@@ -400,13 +430,13 @@ def _build_symbol_candidates(
# predating the field must not silently select a wider slice than the
# declared default.
pct = getattr(inputs.config, "top_symbol_percentile", 0.10) or 0.0
- if pct <= 0:
- return []
# At least one, so a repo with few public symbols still gets a spotlight.
keep = len(deduped) if pct >= 1.0 else max(1, int(len(deduped) * pct))
+ if pct <= 0:
+ keep = 0
# Sized on the whole pool so production never loses a slot it had; tests only
# hand theirs to the next production symbol.
- return [c for c in deduped if c[1][0] not in test_paths][:keep]
+ return [c for c in deduped if c[1][0] not in test_paths], keep
def _layer_map_from_kg(
diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.py
index e41af09f6..0aca27748 100644
--- a/packages/core/src/repowise/core/persistence/models.py
+++ b/packages/core/src/repowise/core/persistence/models.py
@@ -257,7 +257,10 @@ class PageVersion(Base):
__tablename__ = "wiki_page_versions"
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_new_uuid)
- page_id: Mapped[str] = mapped_column(Text, ForeignKey("wiki_pages.id"), nullable=False)
+ # Indexed so deleting a page does not scan every version for the FK check.
+ page_id: Mapped[str] = mapped_column(
+ Text, ForeignKey("wiki_pages.id"), nullable=False, index=True
+ )
repository_id: Mapped[str] = mapped_column(String(32), nullable=False)
version: Mapped[int] = mapped_column(Integer, nullable=False)
page_type: Mapped[str] = mapped_column(String(64), nullable=False)
diff --git a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
index 853eef2fb..951b9813a 100644
--- a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
+++ b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
@@ -54,6 +54,7 @@ class IncrementalPersist(PersistRun):
tombstoned_page_ids: list[str] = field(default_factory=list)
# Same contract, for rows of a page that has been retired outright.
swept_page_ids: list[str] = field(default_factory=list)
+ deselected_page_ids: list[str] = field(default_factory=list)
fts_cleanup_ids: set[str] = field(default_factory=set)
vector_cleanup_ids: set[str] = field(default_factory=set)
@@ -101,6 +102,7 @@ async def sweep_retired(ctx: IncrementalPersist) -> None:
from repowise.core.pipeline.persist import (
sweep_absent_cycle_pages,
sweep_absent_symbol_spotlights,
+ sweep_deselected_pages,
sweep_retired_pages,
)
@@ -117,6 +119,23 @@ async def sweep_retired(ctx: IncrementalPersist) -> None:
ctx.swept_page_ids += await sweep_absent_symbol_spotlights(
ctx.session, ctx.repo_id, ctx.graph_builder
)
+ # Unlike the pages above, these may carry embeddings.
+ ctx.deselected_page_ids = await sweep_deselected_pages(
+ ctx.session,
+ ctx.repo_id,
+ ctx.parsed_files,
+ ctx.graph_builder,
+ _generation_config(ctx.repo_path),
+ ctx.git_meta_map,
+ )
+ ctx.swept_page_ids += ctx.deselected_page_ids
+
+
+def _generation_config(repo_path: Any) -> Any:
+ from repowise.core.generation import GenerationConfig
+ from repowise.core.repo_config import load_repo_config
+
+ return GenerationConfig.from_repo_config(load_repo_config(Path(repo_path)))
async def mark_tombstones(ctx: IncrementalPersist) -> None:
@@ -626,7 +645,9 @@ async def _write(ctx: IncrementalPersist) -> None:
ctx.fts_cleanup_ids = (
set(ctx.tombstoned_page_ids) | set(ctx.swept_page_ids) | cleanup_debt["fts"]
)
- ctx.vector_cleanup_ids = set(ctx.tombstoned_page_ids) | cleanup_debt["vectors"]
+ ctx.vector_cleanup_ids = (
+ set(ctx.tombstoned_page_ids) | set(ctx.deselected_page_ids) | cleanup_debt["vectors"]
+ )
await run_steps(ctx, CLEANUP_STEPS)
# Ceiling: retired non-file pages' *vector* embeddings survive this path.
diff --git a/packages/core/src/repowise/core/pipeline/persist/__init__.py b/packages/core/src/repowise/core/pipeline/persist/__init__.py
index 4eb71bbb3..25e3d44ce 100644
--- a/packages/core/src/repowise/core/pipeline/persist/__init__.py
+++ b/packages/core/src/repowise/core/pipeline/persist/__init__.py
@@ -84,6 +84,7 @@ from .sweeps import (
_symbol_names_by_file,
sweep_absent_cycle_pages,
sweep_absent_symbol_spotlights,
+ sweep_deselected_pages,
sweep_retired_pages,
sweep_superseded_generated_pages,
)
diff --git a/packages/core/src/repowise/core/pipeline/persist/sweeps.py b/packages/core/src/repowise/core/pipeline/persist/sweeps.py
index f952f89ac..e48625eec 100644
--- a/packages/core/src/repowise/core/pipeline/persist/sweeps.py
+++ b/packages/core/src/repowise/core/pipeline/persist/sweeps.py
@@ -323,6 +323,116 @@ async def sweep_absent_symbol_spotlights(
return stale
+# Hysteresis: a spotlight is retired only past this multiple of the selection's cut.
+SPOTLIGHT_RETIRE_MARGIN = 1.5
+
+
+async def sweep_deselected_pages(
+ session: Any,
+ repo_id: str,
+ parsed_files: list[Any] | None,
+ graph_builder: Any,
+ config: Any,
+ git_meta_map: dict | None = None,
+) -> list[str]:
+ """Delete template file pages and spotlights the selection no longer picks.
+
+ An update renders only changed files, so nothing else revisits these. Only
+ files in this run's parse are judged; model-written pages are left alone.
+ *git_meta_map* (this run's rows) overrides the stored hotspot flags.
+ """
+ from sqlalchemy import delete, select
+
+ from repowise.core.generation.page_generator.helpers import _select_clone_representatives
+ from repowise.core.generation.selection.selector import (
+ SelectionInputs,
+ _is_code_file,
+ file_page_eligible,
+ rank_symbol_candidates,
+ )
+ from repowise.core.persistence.models import GitMetadata, Page, PageVersion
+
+ parsed_by_path = {p.file_info.path: p for p in parsed_files or []}
+ if not parsed_by_path or graph_builder is None:
+ return []
+ try:
+ pagerank = graph_builder.pagerank()
+ except Exception: # a released graph cannot rank anything
+ return []
+
+ rows = (
+ await session.execute(
+ select(Page.id, Page.page_type, Page.target_path).where(
+ Page.repository_id == repo_id,
+ Page.page_type.in_(("file_page", "symbol_spotlight")),
+ Page.freshness_status != "tombstone",
+ Page.provider_name == "template",
+ )
+ )
+ ).all()
+ file_rows = [(pid, t) for pid, kind, t in rows if kind == "file_page" and t in parsed_by_path]
+ spot_rows = [(pid, t) for pid, kind, t in rows if kind == "symbol_spotlight"]
+
+ retired: list[str] = []
+ if file_rows:
+ hotspots = set(
+ (
+ await session.execute(
+ select(GitMetadata.file_path).where(
+ GitMetadata.repository_id == repo_id,
+ GitMetadata.is_hotspot.is_(True),
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for path, meta in (git_meta_map or {}).items():
+ if meta.get("is_hotspot"):
+ hotspots.add(path)
+ else:
+ hotspots.discard(path)
+ retired += [
+ pid
+ for pid, path in file_rows
+ if not file_page_eligible(parsed_by_path[path], is_hotspot=path in hotspots)
+ ]
+
+ if spot_rows:
+ # The same candidate pool generation ranks: near-clone losers out first.
+ parsed = list(parsed_by_path.values())
+ if getattr(config, "dedupe_near_clones", True):
+ losers = _select_clone_representatives(
+ [p for p in parsed if _is_code_file(p)], pagerank
+ )
+ parsed = [p for p in parsed if p.file_info.path not in losers]
+ ranked, keep = rank_symbol_candidates(
+ SelectionInputs(
+ parsed_files=parsed,
+ pagerank=pagerank,
+ betweenness={},
+ community={},
+ community_info=None,
+ sccs=[],
+ git_meta_map={},
+ config=config,
+ )
+ )
+ kept = {target for _, target in ranked[: int(keep * SPOTLIGHT_RETIRE_MARGIN)]}
+ for pid, target in spot_rows:
+ path, sep, name = (target or "").partition("::")
+ if sep and path in parsed_by_path and (path, name) not in kept:
+ retired.append(pid)
+
+ for i in range(0, len(retired), _PRUNE_CHUNK):
+ batch = retired[i : i + _PRUNE_CHUNK]
+ await session.execute(delete(PageVersion).where(PageVersion.page_id.in_(batch)))
+ await session.execute(delete(Page).where(Page.repository_id == repo_id, Page.id.in_(batch)))
+ if retired:
+ logger.info("deselected_pages_swept", repo_id=repo_id, count=len(retired))
+ return retired
+
+
async def sweep_superseded_generated_pages(
session: Any,
repo_id: str,
diff --git a/tests/unit/_persist_step_recorders.py b/tests/unit/_persist_step_recorders.py
index 574fa9288..0ae1b53d5 100644
--- a/tests/unit/_persist_step_recorders.py
+++ b/tests/unit/_persist_step_recorders.py
@@ -119,6 +119,9 @@ def install_shared_steps(rec: StepRecorder, *, prune_ret: tuple[int, list[str]]
rec.patch(persist, "sweep_retired_pages", ret=[SWEPT])
rec.patch(persist, "sweep_absent_cycle_pages", ret=[])
rec.patch(persist, "sweep_absent_symbol_spotlights", ret=[])
+ rec.patch(persist, "sweep_deselected_pages", ret=[])
+ # One value for both persisters, so their sweep inputs compare equal.
+ rec.monkeypatch.setattr(persist_index, "_generation_config", lambda _path: "config")
rec.patch(persist, "tombstone_candidates", ret=[], sync=True)
rec.patch(persist, "mark_tombstone_pages", ret=[TOMBSTONED])
rec.patch(page_tree_sync, "rebuild_page_tree")
diff --git a/tests/unit/cli/test_full_update_persist_step_order.py b/tests/unit/cli/test_full_update_persist_step_order.py
index a5cb66f62..861b68ef8 100644
--- a/tests/unit/cli/test_full_update_persist_step_order.py
+++ b/tests/unit/cli/test_full_update_persist_step_order.py
@@ -47,6 +47,7 @@ HAPPY_ORDER = [
"sweep_retired_pages",
"sweep_absent_cycle_pages",
"sweep_absent_symbol_spotlights",
+ "sweep_deselected_pages",
# Swept embeddings leave before the SQL session commits.
"vectors.delete_many",
"tombstone_candidates",
@@ -377,6 +378,7 @@ SHARED_STEPS = [
"sweep_retired_pages",
"sweep_absent_cycle_pages",
"sweep_absent_symbol_spotlights",
+ "sweep_deselected_pages",
"tombstone_candidates",
"mark_tombstone_pages",
"rebuild_page_tree",
diff --git a/tests/unit/persistence/test_deselected_page_sweep.py b/tests/unit/persistence/test_deselected_page_sweep.py
new file mode 100644
index 000000000..bf7dd366e
--- /dev/null
+++ b/tests/unit/persistence/test_deselected_page_sweep.py
@@ -0,0 +1,191 @@
+"""`repowise update` retires template pages the selection no longer picks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import pytest
+
+from repowise.core.generation.models import GenerationConfig, compute_page_id
+from repowise.core.generation.selection.selector import file_page_eligible
+from repowise.core.persistence.models import Page
+from repowise.core.pipeline.persist import sweep_deselected_pages
+from tests.unit.persistence.helpers import insert_repo, make_page_kwargs
+
+
+@dataclass
+class _FileInfo:
+ path: str
+ language: str = "python"
+ size_bytes: int = 5_000
+ is_test: bool = False
+ is_api_contract: bool = False
+ is_entry_point: bool = False
+
+
+@dataclass
+class _Symbol:
+ name: str
+ kind: str = "function"
+ visibility: str = "public"
+
+
+@dataclass
+class _Parsed:
+ file_info: _FileInfo
+ symbols: list[_Symbol] = field(default_factory=list)
+
+
+class _Builder:
+ def __init__(self, pagerank: dict[str, float]) -> None:
+ self._pagerank = pagerank
+
+ def pagerank(self) -> dict[str, float]:
+ return self._pagerank
+
+
+# Twenty production files with one distinct public function each, ranked by
+# PageRank: the default 10% keeps two spotlights, the margin protects three.
+_RANKED = [f"src/f{i:02d}.py" for i in range(20)]
+
+
+def _parsed() -> list[_Parsed]:
+ files = [_Parsed(_FileInfo(p), [_Symbol(f"fn{i}")]) for i, p in enumerate(_RANKED)]
+ files.append(_Parsed(_FileInfo("tests/test_a.py", is_test=True), [_Symbol("test_a")]))
+ files.append(_Parsed(_FileInfo("src/shim.py")))
+ return files
+
+
+def _builder() -> _Builder:
+ return _Builder({p: 1.0 - i / 100 for i, p in enumerate(_RANKED)})
+
+
+async def repo_id(async_session) -> str:
+ return (await insert_repo(async_session)).id
+
+
+async def _add(async_session, repo_id: str, page_type: str, target: str, **kw) -> str:
+ from repowise.core.persistence.crud import upsert_page
+
+ page_id = compute_page_id(page_type, target)
+ await upsert_page(
+ async_session,
+ **make_page_kwargs(
+ repo_id,
+ page_id=page_id,
+ page_type=page_type,
+ target_path=target,
+ provider_name=kw.get("provider_name", "template"),
+ ),
+ )
+ await async_session.flush()
+ return page_id
+
+
+async def _ids(async_session) -> set[str]:
+ return {r.id for r in (await async_session.execute(Page.__table__.select())).fetchall()}
+
+
+async def _sweep(async_session, repo_id: str) -> list[str]:
+ return await sweep_deselected_pages(
+ async_session, repo_id, _parsed(), _builder(), GenerationConfig()
+ )
+
+
+class TestDeselectedFilePages:
+ async def test_test_file_and_shim_pages_are_retired(self, async_session, repo_id) -> None:
+ test_page = await _add(async_session, repo_id, "file_page", "tests/test_a.py")
+ shim = await _add(async_session, repo_id, "file_page", "src/shim.py")
+ kept = await _add(async_session, repo_id, "file_page", "src/f10.py")
+
+ swept = await _sweep(async_session, repo_id)
+
+ assert sorted(swept) == sorted([test_page, shim])
+ assert await _ids(async_session) == {kept}
+
+ async def test_unparsed_file_keeps_its_page(self, async_session, repo_id) -> None:
+ page = await _add(async_session, repo_id, "file_page", "tests/excluded_test.py")
+ assert await _sweep(async_session, repo_id) == []
+ assert await _ids(async_session) == {page}
+
+ async def test_this_runs_hotspot_flag_keeps_a_symbol_less_page(
+ self, async_session, repo_id
+ ) -> None:
+ shim = await _add(async_session, repo_id, "file_page", "src/shim.py")
+ swept = await sweep_deselected_pages(
+ async_session,
+ repo_id,
+ _parsed(),
+ _builder(),
+ GenerationConfig(),
+ {"src/shim.py": {"is_hotspot": True}},
+ )
+ assert swept == []
+ assert await _ids(async_session) == {shim}
+
+ async def test_model_written_page_is_kept(self, async_session, repo_id) -> None:
+ page = await _add(
+ async_session, repo_id, "file_page", "tests/test_a.py", provider_name="openai"
+ )
+ assert await _sweep(async_session, repo_id) == []
+ assert await _ids(async_session) == {page}
+
+
+class TestDeselectedSpotlights:
+ async def test_far_outside_the_cut_is_retired(self, async_session, repo_id) -> None:
+ top = await _add(async_session, repo_id, "symbol_spotlight", "src/f00.py::fn0")
+ # Rank 3: past the 10% cut, inside the margin, so it is kept.
+ boundary = await _add(async_session, repo_id, "symbol_spotlight", "src/f02.py::fn2")
+ out = await _add(async_session, repo_id, "symbol_spotlight", "src/f05.py::fn5")
+ test_sym = await _add(
+ async_session, repo_id, "symbol_spotlight", "tests/test_a.py::test_a"
+ )
+
+ swept = await _sweep(async_session, repo_id)
+
+ assert sorted(swept) == sorted([out, test_sym])
+ assert await _ids(async_session) == {top, boundary}
+
+ async def test_near_clone_loser_is_retired(self, async_session, repo_id) -> None:
+ # Three same-shaped files in one directory: generation keeps the top one.
+ clones = [_Parsed(_FileInfo(f"gen/c{i}.py"), [_Symbol("run")]) for i in range(3)]
+ builder = _Builder({"gen/c0.py": 1.0, "gen/c1.py": 0.9, "gen/c2.py": 0.8})
+ kept = await _add(async_session, repo_id, "symbol_spotlight", "gen/c0.py::run")
+ loser = await _add(async_session, repo_id, "symbol_spotlight", "gen/c1.py::run")
+
+ swept = await sweep_deselected_pages(
+ async_session, repo_id, clones, builder, GenerationConfig(top_symbol_percentile=1.0)
+ )
+
+ assert swept == [loser]
+ assert await _ids(async_session) == {kept}
+
+ async def test_unparsed_file_keeps_its_spotlight(self, async_session, repo_id) -> None:
+ page = await _add(async_session, repo_id, "symbol_spotlight", "gone.py::f")
+ assert await _sweep(async_session, repo_id) == []
+ assert await _ids(async_session) == {page}
+
+ async def test_missing_graph_is_a_no_op(self, async_session, repo_id) -> None:
+ page = await _add(async_session, repo_id, "symbol_spotlight", "src/f05.py::fn5")
+ swept = await sweep_deselected_pages(
+ async_session, repo_id, _parsed(), None, GenerationConfig()
+ )
+ assert swept == []
+ assert await _ids(async_session) == {page}
+
+
+def test_file_page_eligible_matches_the_selection_floors() -> None:
+ code = _Parsed(_FileInfo("src/a.py"), [_Symbol("f")])
+ shim = _Parsed(_FileInfo("src/b.py"))
+ test = _Parsed(_FileInfo("tests/test_c.py", is_test=True), [_Symbol("t")])
+ init = _Parsed(_FileInfo("src/pkg/__init__.py"), [_Symbol("x"), _Symbol("y")])
+
+ assert file_page_eligible(code, is_hotspot=False)
+ assert not file_page_eligible(shim, is_hotspot=False)
+ # A hotspot earns a page even with no symbols, as selection scores it.
+ assert file_page_eligible(shim, is_hotspot=True)
+ assert not file_page_eligible(test, is_hotspot=False)
+ assert not file_page_eligible(init, is_hotspot=False)
diff --git a/tests/unit/pipeline/test_incremental_persist_step_order.py b/tests/unit/pipeline/test_incremental_persist_step_order.py
index 9ea1ae286..bd758f58f 100644
--- a/tests/unit/pipeline/test_incremental_persist_step_order.py
+++ b/tests/unit/pipeline/test_incremental_persist_step_order.py
@@ -32,6 +32,7 @@ HAPPY_ORDER = [
"sweep_retired_pages",
"sweep_absent_cycle_pages",
"sweep_absent_symbol_spotlights",
+ "sweep_deselected_pages",
"tombstone_candidates",
"mark_tombstone_pages",
"rebuild_page_tree",