Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(update): retire template pages the selection no longer picks (#4211)
Raghav Chamadiyabdebde6b5dc2
diff --git a/packages/core/alembic/versions/0116_page_versions_page_id_index.py b/packages/core/alembic/versions/0116_page_versions_page_id_index.pynew file mode 100644index 000000000..4b94d121e--- /dev/null+++ b/packages/core/alembic/versions/0116_page_versions_page_id_index.py@@ -0,0 +1,28 @@+"""Index wiki_page_versions.page_id, the foreign key every page delete checks.++Local SQLite gets the index from ``init_db``.++Revision ID: 0116+Revises: 0115+Create Date: 2026-10-19+"""++from __future__ import annotations++from collections.abc import Sequence++from alembic import op++# revision identifiers+revision: str = "0116"+down_revision: str | None = "0115"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.create_index("ix_wiki_page_versions_page_id", "wiki_page_versions", ["page_id"])+++def downgrade() -> None:+ op.drop_index("ix_wiki_page_versions_page_id", table_name="wiki_page_versions")diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.pyindex d58bd50a7..462080e24 100644--- a/packages/core/src/repowise/core/generation/selection/selector.py+++ b/packages/core/src/repowise/core/generation/selection/selector.py@@ -283,6 +283,28 @@ def count_documentable_files(parsed_files: list[Any]) -> int:return sum(1 for p in parsed_files if _is_code_file(p) and _passes_importance_floor(p))+def file_page_eligible(parsed: Any, *, is_hotspot: bool) -> bool:+ """Whether *parsed* clears every per-file gate of file-page selection.++ Repo-wide gates (cap, near-clone dedupe) are not asked: an update rendering+ the file alone would still give it a page.+ """+ return (+ _is_code_file(parsed)+ and _passes_importance_floor(parsed)+ # Graph metrics only add to a positive score.+ and score_file(+ parsed,+ pagerank=0.0,+ betweenness=0.0,+ max_pagerank=0.0,+ max_betweenness=0.0,+ is_hotspot=is_hotspot,+ )+ > 0.0+ )++def _passes_importance_floor(parsed: Any) -> bool:"""Whether *parsed* is worth a file page at all.@@ -365,6 +387,14 @@ def _build_symbol_candidates(inputs: SelectionInputs,) -> list[tuple[float, tuple[str, str]]]:"""Return ``[(score, (file_path, symbol_name)), ...]`` descending."""+ ranked, keep = rank_symbol_candidates(inputs)+ return ranked[:keep]+++def rank_symbol_candidates(+ inputs: SelectionInputs,+) -> tuple[list[tuple[float, tuple[str, str]]], int]:+ """Every production spotlight candidate in rank order, and how many selection keeps."""max_pr = max(inputs.pagerank.values(), default=0.0)test_paths = {p.file_info.path for p in inputs.parsed_files if p.file_info.is_test}scored: list[tuple[float, tuple[str, str]]] = []@@ -400,13 +430,13 @@ def _build_symbol_candidates(# predating the field must not silently select a wider slice than the# declared default.pct = getattr(inputs.config, "top_symbol_percentile", 0.10) or 0.0- if pct <= 0:- return []# At least one, so a repo with few public symbols still gets a spotlight.keep = len(deduped) if pct >= 1.0 else max(1, int(len(deduped) * pct))+ if pct <= 0:+ keep = 0# Sized on the whole pool so production never loses a slot it had; tests only# hand theirs to the next production symbol.- return [c for c in deduped if c[1][0] not in test_paths][:keep]+ return [c for c in deduped if c[1][0] not in test_paths], keepdef _layer_map_from_kg(diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.pyindex e41af09f6..0aca27748 100644--- a/packages/core/src/repowise/core/persistence/models.py+++ b/packages/core/src/repowise/core/persistence/models.py@@ -257,7 +257,10 @@ class PageVersion(Base):__tablename__ = "wiki_page_versions"id: Mapped[str] = mapped_column(String(32), primary_key=True, default=_new_uuid)- page_id: Mapped[str] = mapped_column(Text, ForeignKey("wiki_pages.id"), nullable=False)+ # Indexed so deleting a page does not scan every version for the FK check.+ page_id: Mapped[str] = mapped_column(+ Text, ForeignKey("wiki_pages.id"), nullable=False, index=True+ )repository_id: Mapped[str] = mapped_column(String(32), nullable=False)version: Mapped[int] = mapped_column(Integer, nullable=False)page_type: Mapped[str] = mapped_column(String(64), nullable=False)diff --git a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py b/packages/core/src/repowise/core/pipeline/incremental/persist_index.pyindex 853eef2fb..951b9813a 100644--- a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py+++ b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py@@ -54,6 +54,7 @@ class IncrementalPersist(PersistRun):tombstoned_page_ids: list[str] = field(default_factory=list)# Same contract, for rows of a page that has been retired outright.swept_page_ids: list[str] = field(default_factory=list)+ deselected_page_ids: list[str] = field(default_factory=list)fts_cleanup_ids: set[str] = field(default_factory=set)vector_cleanup_ids: set[str] = field(default_factory=set)@@ -101,6 +102,7 @@ async def sweep_retired(ctx: IncrementalPersist) -> None:from repowise.core.pipeline.persist import (sweep_absent_cycle_pages,sweep_absent_symbol_spotlights,+ sweep_deselected_pages,sweep_retired_pages,)@@ -117,6 +119,23 @@ async def sweep_retired(ctx: IncrementalPersist) -> None:ctx.swept_page_ids += await sweep_absent_symbol_spotlights(ctx.session, ctx.repo_id, ctx.graph_builder)+ # Unlike the pages above, these may carry embeddings.+ ctx.deselected_page_ids = await sweep_deselected_pages(+ ctx.session,+ ctx.repo_id,+ ctx.parsed_files,+ ctx.graph_builder,+ _generation_config(ctx.repo_path),+ ctx.git_meta_map,+ )+ ctx.swept_page_ids += ctx.deselected_page_ids+++def _generation_config(repo_path: Any) -> Any:+ from repowise.core.generation import GenerationConfig+ from repowise.core.repo_config import load_repo_config++ return GenerationConfig.from_repo_config(load_repo_config(Path(repo_path)))async def mark_tombstones(ctx: IncrementalPersist) -> None:@@ -626,7 +645,9 @@ async def _write(ctx: IncrementalPersist) -> None:ctx.fts_cleanup_ids = (set(ctx.tombstoned_page_ids) | set(ctx.swept_page_ids) | cleanup_debt["fts"])- ctx.vector_cleanup_ids = set(ctx.tombstoned_page_ids) | cleanup_debt["vectors"]+ ctx.vector_cleanup_ids = (+ set(ctx.tombstoned_page_ids) | set(ctx.deselected_page_ids) | cleanup_debt["vectors"]+ )await run_steps(ctx, CLEANUP_STEPS)# Ceiling: retired non-file pages' *vector* embeddings survive this path.diff --git a/packages/core/src/repowise/core/pipeline/persist/__init__.py b/packages/core/src/repowise/core/pipeline/persist/__init__.pyindex 4eb71bbb3..25e3d44ce 100644--- a/packages/core/src/repowise/core/pipeline/persist/__init__.py+++ b/packages/core/src/repowise/core/pipeline/persist/__init__.py@@ -84,6 +84,7 @@ from .sweeps import (_symbol_names_by_file,sweep_absent_cycle_pages,sweep_absent_symbol_spotlights,+ sweep_deselected_pages,sweep_retired_pages,sweep_superseded_generated_pages,)diff --git a/packages/core/src/repowise/core/pipeline/persist/sweeps.py b/packages/core/src/repowise/core/pipeline/persist/sweeps.pyindex f952f89ac..e48625eec 100644--- a/packages/core/src/repowise/core/pipeline/persist/sweeps.py+++ b/packages/core/src/repowise/core/pipeline/persist/sweeps.py@@ -323,6 +323,116 @@ async def sweep_absent_symbol_spotlights(return stale+# Hysteresis: a spotlight is retired only past this multiple of the selection's cut.+SPOTLIGHT_RETIRE_MARGIN = 1.5+++async def sweep_deselected_pages(+ session: Any,+ repo_id: str,+ parsed_files: list[Any] | None,+ graph_builder: Any,+ config: Any,+ git_meta_map: dict | None = None,+) -> list[str]:+ """Delete template file pages and spotlights the selection no longer picks.++ An update renders only changed files, so nothing else revisits these. Only+ files in this run's parse are judged; model-written pages are left alone.+ *git_meta_map* (this run's rows) overrides the stored hotspot flags.+ """+ from sqlalchemy import delete, select++ from repowise.core.generation.page_generator.helpers import _select_clone_representatives+ from repowise.core.generation.selection.selector import (+ SelectionInputs,+ _is_code_file,+ file_page_eligible,+ rank_symbol_candidates,+ )+ from repowise.core.persistence.models import GitMetadata, Page, PageVersion++ parsed_by_path = {p.file_info.path: p for p in parsed_files or []}+ if not parsed_by_path or graph_builder is None:+ return []+ try:+ pagerank = graph_builder.pagerank()+ except Exception: # a released graph cannot rank anything+ return []++ rows = (+ await session.execute(+ select(Page.id, Page.page_type, Page.target_path).where(+ Page.repository_id == repo_id,+ Page.page_type.in_(("file_page", "symbol_spotlight")),+ Page.freshness_status != "tombstone",+ Page.provider_name == "template",+ )+ )+ ).all()+ file_rows = [(pid, t) for pid, kind, t in rows if kind == "file_page" and t in parsed_by_path]+ spot_rows = [(pid, t) for pid, kind, t in rows if kind == "symbol_spotlight"]++ retired: list[str] = []+ if file_rows:+ hotspots = set(+ (+ await session.execute(+ select(GitMetadata.file_path).where(+ GitMetadata.repository_id == repo_id,+ GitMetadata.is_hotspot.is_(True),+ )+ )+ )+ .scalars()+ .all()+ )+ for path, meta in (git_meta_map or {}).items():+ if meta.get("is_hotspot"):+ hotspots.add(path)+ else:+ hotspots.discard(path)+ retired += [+ pid+ for pid, path in file_rows+ if not file_page_eligible(parsed_by_path[path], is_hotspot=path in hotspots)+ ]++ if spot_rows:+ # The same candidate pool generation ranks: near-clone losers out first.+ parsed = list(parsed_by_path.values())+ if getattr(config, "dedupe_near_clones", True):+ losers = _select_clone_representatives(+ [p for p in parsed if _is_code_file(p)], pagerank+ )+ parsed = [p for p in parsed if p.file_info.path not in losers]+ ranked, keep = rank_symbol_candidates(+ SelectionInputs(+ parsed_files=parsed,+ pagerank=pagerank,+ betweenness={},+ community={},+ community_info=None,+ sccs=[],+ git_meta_map={},+ config=config,+ )+ )+ kept = {target for _, target in ranked[: int(keep * SPOTLIGHT_RETIRE_MARGIN)]}+ for pid, target in spot_rows:+ path, sep, name = (target or "").partition("::")+ if sep and path in parsed_by_path and (path, name) not in kept:+ retired.append(pid)++ for i in range(0, len(retired), _PRUNE_CHUNK):+ batch = retired[i : i + _PRUNE_CHUNK]+ await session.execute(delete(PageVersion).where(PageVersion.page_id.in_(batch)))+ await session.execute(delete(Page).where(Page.repository_id == repo_id, Page.id.in_(batch)))+ if retired:+ logger.info("deselected_pages_swept", repo_id=repo_id, count=len(retired))+ return retired++async def sweep_superseded_generated_pages(session: Any,repo_id: str,diff --git a/tests/unit/_persist_step_recorders.py b/tests/unit/_persist_step_recorders.pyindex 574fa9288..0ae1b53d5 100644--- a/tests/unit/_persist_step_recorders.py+++ b/tests/unit/_persist_step_recorders.py@@ -119,6 +119,9 @@ def install_shared_steps(rec: StepRecorder, *, prune_ret: tuple[int, list[str]]rec.patch(persist, "sweep_retired_pages", ret=[SWEPT])rec.patch(persist, "sweep_absent_cycle_pages", ret=[])rec.patch(persist, "sweep_absent_symbol_spotlights", ret=[])+ rec.patch(persist, "sweep_deselected_pages", ret=[])+ # One value for both persisters, so their sweep inputs compare equal.+ rec.monkeypatch.setattr(persist_index, "_generation_config", lambda _path: "config")rec.patch(persist, "tombstone_candidates", ret=[], sync=True)rec.patch(persist, "mark_tombstone_pages", ret=[TOMBSTONED])rec.patch(page_tree_sync, "rebuild_page_tree")diff --git a/tests/unit/cli/test_full_update_persist_step_order.py b/tests/unit/cli/test_full_update_persist_step_order.pyindex a5cb66f62..861b68ef8 100644--- a/tests/unit/cli/test_full_update_persist_step_order.py+++ b/tests/unit/cli/test_full_update_persist_step_order.py@@ -47,6 +47,7 @@ HAPPY_ORDER = ["sweep_retired_pages","sweep_absent_cycle_pages","sweep_absent_symbol_spotlights",+ "sweep_deselected_pages",# Swept embeddings leave before the SQL session commits."vectors.delete_many","tombstone_candidates",@@ -377,6 +378,7 @@ SHARED_STEPS = ["sweep_retired_pages","sweep_absent_cycle_pages","sweep_absent_symbol_spotlights",+ "sweep_deselected_pages","tombstone_candidates","mark_tombstone_pages","rebuild_page_tree",diff --git a/tests/unit/persistence/test_deselected_page_sweep.py b/tests/unit/persistence/test_deselected_page_sweep.pynew file mode 100644index 000000000..bf7dd366e--- /dev/null+++ b/tests/unit/persistence/test_deselected_page_sweep.py@@ -0,0 +1,191 @@+"""`repowise update` retires template pages the selection no longer picks."""++from __future__ import annotations++from dataclasses import dataclass, field++import pytest++from repowise.core.generation.models import GenerationConfig, compute_page_id+from repowise.core.generation.selection.selector import file_page_eligible+from repowise.core.persistence.models import Page+from repowise.core.pipeline.persist import sweep_deselected_pages+from tests.unit.persistence.helpers import insert_repo, make_page_kwargs+++@dataclass+class _FileInfo:+ path: str+ language: str = "python"+ size_bytes: int = 5_000+ is_test: bool = False+ is_api_contract: bool = False+ is_entry_point: bool = False+++@dataclass+class _Symbol:+ name: str+ kind: str = "function"+ visibility: str = "public"+++@dataclass+class _Parsed:+ file_info: _FileInfo+ symbols: list[_Symbol] = field(default_factory=list)+++class _Builder:+ def __init__(self, pagerank: dict[str, float]) -> None:+ self._pagerank = pagerank++ def pagerank(self) -> dict[str, float]:+ return self._pagerank+++# Twenty production files with one distinct public function each, ranked by+# PageRank: the default 10% keeps two spotlights, the margin protects three.+_RANKED = [f"src/f{i:02d}.py" for i in range(20)]+++def _parsed() -> list[_Parsed]:+ files = [_Parsed(_FileInfo(p), [_Symbol(f"fn{i}")]) for i, p in enumerate(_RANKED)]+ files.append(_Parsed(_FileInfo("tests/test_a.py", is_test=True), [_Symbol("test_a")]))+ files.append(_Parsed(_FileInfo("src/shim.py")))+ return files+++def _builder() -> _Builder:+ return _Builder({p: 1.0 - i / 100 for i, p in enumerate(_RANKED)})+++async def repo_id(async_session) -> str:+ return (await insert_repo(async_session)).id+++async def _add(async_session, repo_id: str, page_type: str, target: str, **kw) -> str:+ from repowise.core.persistence.crud import upsert_page++ page_id = compute_page_id(page_type, target)+ await upsert_page(+ async_session,+ **make_page_kwargs(+ repo_id,+ page_id=page_id,+ page_type=page_type,+ target_path=target,+ provider_name=kw.get("provider_name", "template"),+ ),+ )+ await async_session.flush()+ return page_id+++async def _ids(async_session) -> set[str]:+ return {r.id for r in (await async_session.execute(Page.__table__.select())).fetchall()}+++async def _sweep(async_session, repo_id: str) -> list[str]:+ return await sweep_deselected_pages(+ async_session, repo_id, _parsed(), _builder(), GenerationConfig()+ )+++class TestDeselectedFilePages:+ async def test_test_file_and_shim_pages_are_retired(self, async_session, repo_id) -> None:+ test_page = await _add(async_session, repo_id, "file_page", "tests/test_a.py")+ shim = await _add(async_session, repo_id, "file_page", "src/shim.py")+ kept = await _add(async_session, repo_id, "file_page", "src/f10.py")++ swept = await _sweep(async_session, repo_id)++ assert sorted(swept) == sorted([test_page, shim])+ assert await _ids(async_session) == {kept}++ async def test_unparsed_file_keeps_its_page(self, async_session, repo_id) -> None:+ page = await _add(async_session, repo_id, "file_page", "tests/excluded_test.py")+ assert await _sweep(async_session, repo_id) == []+ assert await _ids(async_session) == {page}++ async def test_this_runs_hotspot_flag_keeps_a_symbol_less_page(+ self, async_session, repo_id+ ) -> None:+ shim = await _add(async_session, repo_id, "file_page", "src/shim.py")+ swept = await sweep_deselected_pages(+ async_session,+ repo_id,+ _parsed(),+ _builder(),+ GenerationConfig(),+ {"src/shim.py": {"is_hotspot": True}},+ )+ assert swept == []+ assert await _ids(async_session) == {shim}++ async def test_model_written_page_is_kept(self, async_session, repo_id) -> None:+ page = await _add(+ async_session, repo_id, "file_page", "tests/test_a.py", provider_name="openai"+ )+ assert await _sweep(async_session, repo_id) == []+ assert await _ids(async_session) == {page}+++class TestDeselectedSpotlights:+ async def test_far_outside_the_cut_is_retired(self, async_session, repo_id) -> None:+ top = await _add(async_session, repo_id, "symbol_spotlight", "src/f00.py::fn0")+ # Rank 3: past the 10% cut, inside the margin, so it is kept.+ boundary = await _add(async_session, repo_id, "symbol_spotlight", "src/f02.py::fn2")+ out = await _add(async_session, repo_id, "symbol_spotlight", "src/f05.py::fn5")+ test_sym = await _add(+ async_session, repo_id, "symbol_spotlight", "tests/test_a.py::test_a"+ )++ swept = await _sweep(async_session, repo_id)++ assert sorted(swept) == sorted([out, test_sym])+ assert await _ids(async_session) == {top, boundary}++ async def test_near_clone_loser_is_retired(self, async_session, repo_id) -> None:+ # Three same-shaped files in one directory: generation keeps the top one.+ clones = [_Parsed(_FileInfo(f"gen/c{i}.py"), [_Symbol("run")]) for i in range(3)]+ builder = _Builder({"gen/c0.py": 1.0, "gen/c1.py": 0.9, "gen/c2.py": 0.8})+ kept = await _add(async_session, repo_id, "symbol_spotlight", "gen/c0.py::run")+ loser = await _add(async_session, repo_id, "symbol_spotlight", "gen/c1.py::run")++ swept = await sweep_deselected_pages(+ async_session, repo_id, clones, builder, GenerationConfig(top_symbol_percentile=1.0)+ )++ assert swept == [loser]+ assert await _ids(async_session) == {kept}++ async def test_unparsed_file_keeps_its_spotlight(self, async_session, repo_id) -> None:+ page = await _add(async_session, repo_id, "symbol_spotlight", "gone.py::f")+ assert await _sweep(async_session, repo_id) == []+ assert await _ids(async_session) == {page}++ async def test_missing_graph_is_a_no_op(self, async_session, repo_id) -> None:+ page = await _add(async_session, repo_id, "symbol_spotlight", "src/f05.py::fn5")+ swept = await sweep_deselected_pages(+ async_session, repo_id, _parsed(), None, GenerationConfig()+ )+ assert swept == []+ assert await _ids(async_session) == {page}+++def test_file_page_eligible_matches_the_selection_floors() -> None:+ code = _Parsed(_FileInfo("src/a.py"), [_Symbol("f")])+ shim = _Parsed(_FileInfo("src/b.py"))+ test = _Parsed(_FileInfo("tests/test_c.py", is_test=True), [_Symbol("t")])+ init = _Parsed(_FileInfo("src/pkg/__init__.py"), [_Symbol("x"), _Symbol("y")])++ assert file_page_eligible(code, is_hotspot=False)+ assert not file_page_eligible(shim, is_hotspot=False)+ # A hotspot earns a page even with no symbols, as selection scores it.+ assert file_page_eligible(shim, is_hotspot=True)+ assert not file_page_eligible(test, is_hotspot=False)+ assert not file_page_eligible(init, is_hotspot=False)diff --git a/tests/unit/pipeline/test_incremental_persist_step_order.py b/tests/unit/pipeline/test_incremental_persist_step_order.pyindex 9ea1ae286..bd758f58f 100644--- a/tests/unit/pipeline/test_incremental_persist_step_order.py+++ b/tests/unit/pipeline/test_incremental_persist_step_order.py@@ -32,6 +32,7 @@ HAPPY_ORDER = ["sweep_retired_pages","sweep_absent_cycle_pages","sweep_absent_symbol_spotlights",+ "sweep_deselected_pages","tombstone_candidates","mark_tombstone_pages","rebuild_page_tree",