Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(update): recount the file-page scope on every update (#4220)
Raghav Chamadiyae4e2bf46de2f
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.pyindex c8917a8e3..d3cf236bc 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py@@ -78,6 +78,7 @@ from .persistence import (parser_changed,rescan_security_if_changed,security_scanner_changed,+ stamp_file_page_scope,stamp_head_commit,)from .reporting import (@@ -1761,6 +1762,11 @@ def _docs_update(opts: UpdateOptions, run: UpdateRun) -> UpdateOutcome:with timed(run.timings, "security_rescan"):rescan_security_if_changed(repo_path, run.state, run.exclude_patterns)+ # Saved before the editor files quote it.+ stamp_file_page_scope(+ repo_path, run.state, run.parsed_files, run.graph_builder, run.persisted_git_meta+ )+ save_state(repo_path, run.state)_finish_outcome(repo_path, opts.agents_md, run.degraded, timings=run.timings)_save_docs_state(run, docs)diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex f85ad8a10..645011f25 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -482,6 +482,98 @@ def _stamp_scope(repo_path: Any, new_state: dict, failed_steps: list[str]) -> No)+def stamp_file_page_scope(+ repo_path: Any,+ state: dict,+ parsed_files: list | None,+ graph_builder: Any,+ git_meta_map: dict | None,+) -> None:+ """Recount ``index_scope.file_pages`` from this run's parse and the stored pages.++ Eligibility applies selection's gates except the cap, as init's figure does;+ an unreadable store keeps the previous figures.+ """+ if not parsed_files or graph_builder is None:+ return+ from repowise.core.generation import GenerationConfig+ from repowise.core.generation.selection.selector import eligible_file_page_paths+ from repowise.core.index_scope import file_page_counts, resolve_index_scope, stamp_index_scope+ from repowise.core.repo_config import load_repo_config++ try:+ pagerank = graph_builder.pagerank()+ except Exception as exc: # a released graph cannot rank anything+ log.debug("file_page_scope_skipped", error=str(exc))+ return+ stored = run_async(_load_page_paths_and_hotspots(repo_path))+ if stored is None:+ return+ page_paths, hotspots = stored+ for path, meta in (git_meta_map or {}).items():+ if meta.get("is_hotspot"):+ hotspots.add(path)+ else:+ hotspots.discard(path)+ eligible = eligible_file_page_paths(+ parsed_files,+ pagerank,+ hotspots=hotspots,+ dedupe_near_clones=GenerationConfig.from_repo_config(+ load_repo_config(Path(repo_path))+ ).dedupe_near_clones,+ )+ config = load_config(repo_path)+ stamp_index_scope(+ state,+ config,+ file_pages=file_page_counts(+ configured_cap=resolve_index_scope(state, config)["file_pages"]["configured_cap"],+ eligible=len(eligible),+ generated=len(eligible & page_paths),+ ),+ )+++async def _load_page_paths_and_hotspots(repo_path: Any) -> tuple[set[str], set[str]] | None:+ """Paths with a live file page, and the stored hotspot paths."""+ from sqlalchemy import select++ from repowise.cli.helpers import get_db_url_for_repo+ from repowise.core.persistence import (+ create_engine,+ create_session_factory,+ get_repository_by_path,+ get_session,+ )+ from repowise.core.persistence.models import GitMetadata, Page++ engine = create_engine(get_db_url_for_repo(repo_path))+ try:+ async with get_session(create_session_factory(engine)) as session:+ repo = await get_repository_by_path(session, str(repo_path))+ if repo is None:+ return None+ pages = await session.execute(+ select(Page.target_path).where(+ Page.repository_id == repo.id,+ Page.page_type == "file_page",+ Page.freshness_status != "tombstone",+ )+ )+ hotspots = await session.execute(+ select(GitMetadata.file_path).where(+ GitMetadata.repository_id == repo.id, GitMetadata.is_hotspot.is_(True)+ )+ )+ return set(pages.scalars()), set(hotspots.scalars())+ except Exception as exc:+ log.warning("file_page_scope_read_failed", error=str(exc))+ return None+ finally:+ await engine.dispose()++def _persist_index_only_update(repo_path: Any,graph_builder: Any,@@ -636,6 +728,7 @@ def _persist_index_only_update(timings.stop("run")new_state["phase_timings"] = timings.totals_stamp_scope(repo_path, new_state, failed_steps)+ stamp_file_page_scope(repo_path, new_state, parsed_files, graph_builder, git_meta_map)save_state(repo_path, new_state)elapsed = time.monotonic() - startfrom .reporting import show_index_only_completiondiff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.pyindex 462080e24..2bb55a00c 100644--- a/packages/core/src/repowise/core/generation/selection/selector.py+++ b/packages/core/src/repowise/core/generation/selection/selector.py@@ -305,6 +305,29 @@ def file_page_eligible(parsed: Any, *, is_hotspot: bool) -> bool:)+def eligible_file_page_paths(+ parsed_files: list[Any],+ pagerank: dict[str, float],+ *,+ hotspots: set[str],+ dedupe_near_clones: bool = True,+) -> set[str]:+ """The files selection would consider for a page: every gate but the cap."""+ from ..page_generator.helpers import _select_clone_representatives++ drop = (+ _select_clone_representatives([p for p in parsed_files if _is_code_file(p)], pagerank)+ if dedupe_near_clones+ else set()+ )+ return {+ p.file_info.path+ for p in parsed_files+ if p.file_info.path not in drop+ and file_page_eligible(p, is_hotspot=p.file_info.path in hotspots)+ }++def _passes_importance_floor(parsed: Any) -> bool:"""Whether *parsed* is worth a file page at all.diff --git a/packages/core/src/repowise/core/index_scope.py b/packages/core/src/repowise/core/index_scope.pyindex 7b65cd8ea..4310df701 100644--- a/packages/core/src/repowise/core/index_scope.py+++ b/packages/core/src/repowise/core/index_scope.py@@ -430,9 +430,16 @@ def file_page_scope(*, configured_cap: int | None, eligible: int, generated_pages: list[Any] | None) -> dict[str, int | None]:"""Build configured/effective/achieved file-page figures for one run."""+ generated = sum(1 for page in (generated_pages or []) if page.page_type == "file_page")+ return file_page_counts(configured_cap=configured_cap, eligible=eligible, generated=generated)+++def file_page_counts(+ *, configured_cap: int | None, eligible: int, generated: int+) -> dict[str, int | None]:+ """The file-page figures from counts, for a run that holds no page list."""from repowise.core.generation.selection import auto_file_page_cap- generated = sum(1 for page in (generated_pages or []) if page.page_type == "file_page")effective = configured_capif configured_cap is None:effective = auto_file_page_cap(eligible)diff --git a/tests/unit/cli/test_update_file_page_scope.py b/tests/unit/cli/test_update_file_page_scope.pynew file mode 100644index 000000000..a9dd27137--- /dev/null+++ b/tests/unit/cli/test_update_file_page_scope.py@@ -0,0 +1,168 @@+"""`repowise update` recounts ``index_scope.file_pages`` (#3905)."""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from repowise.cli.commands.update_cmd import persistence+from repowise.core.generation.selection.selector import eligible_file_page_paths+from repowise.core.index_scope import file_page_counts, file_page_scope+++def _parsed(path: str, *, is_test: bool = False, symbols: tuple[str, ...] = ("f",)):+ return SimpleNamespace(+ file_info=SimpleNamespace(+ path=path,+ language="python",+ size_bytes=5_000,+ is_test=is_test,+ is_api_contract=False,+ is_entry_point=False,+ ),+ symbols=[SimpleNamespace(name=n, kind="function") for n in symbols],+ )+++class _Builder:+ def pagerank(self) -> dict[str, float]:+ return {}+++def _stale_state() -> dict:+ pages = {"configured_cap": None, "effective_cap": None, "eligible": 1, "generated": 1}+ return {"index_scope": {"file_pages": {**pages, "omitted": 0}}}+++def no_config(monkeypatch):+ monkeypatch.setattr(persistence, "load_config", lambda _repo: {})+ monkeypatch.setattr("repowise.core.repo_config.load_repo_config", lambda _repo: {})+++def _stored(page_paths: set[str], hotspots: set[str]):+ async def load(_repo):+ return set(page_paths), set(hotspots)++ return load+++def test_recounts_from_parse_and_stored_pages(monkeypatch, no_config) -> None:+ # The test file's leftover page is not counted as generated.+ monkeypatch.setattr(+ persistence,+ "_load_page_paths_and_hotspots",+ _stored({"a.py", "b.py", "t.py"}, set()),+ )+ state = _stale_state()+ parsed = [+ _parsed("a.py", symbols=("fa",)),+ _parsed("b.py", symbols=("fb",)),+ _parsed("c.py", symbols=("fc",)),+ _parsed("t.py", is_test=True),+ ]++ persistence.stamp_file_page_scope("repo", state, parsed, _Builder(), {})++ pages = state["index_scope"]["file_pages"]+ assert (pages["eligible"], pages["generated"], pages["omitted"]) == (3, 2, 1)+++def test_this_runs_hotspot_rows_win(monkeypatch, no_config) -> None:+ monkeypatch.setattr(+ persistence, "_load_page_paths_and_hotspots", _stored(set(), {"shim.py"})+ )+ state = _stale_state()+ parsed = [_parsed("shim.py", symbols=()), _parsed("other.py", symbols=())]++ rows = {"shim.py": {"is_hotspot": False}, "other.py": {"is_hotspot": True}}+ persistence.stamp_file_page_scope("repo", state, parsed, _Builder(), rows)++ pages = state["index_scope"]["file_pages"]+ assert (pages["eligible"], pages["omitted"]) == (1, 1)+++def test_explicit_unlimited_cap_is_kept(monkeypatch, no_config) -> None:+ monkeypatch.setattr(persistence, "_load_page_paths_and_hotspots", _stored({"a.py"}, set()))+ state = _stale_state()+ state["index_scope"]["file_pages"]["configured_cap"] = 0++ persistence.stamp_file_page_scope("repo", state, [_parsed("a.py")], _Builder(), {})++ pages = state["index_scope"]["file_pages"]+ assert (pages["configured_cap"], pages["effective_cap"], pages["omitted"]) == (0, None, 0)+++def test_docs_update_stamps_before_the_editor_files_quote_it() -> None:+ import inspect++ from repowise.cli.commands.update_cmd.command import _docs_update++ source = inspect.getsource(_docs_update)+ assert source.index("stamp_file_page_scope") < source.index("_finish_outcome")+++def test_unreadable_store_keeps_previous_figures(monkeypatch, no_config) -> None:+ async def load(_repo):+ return None++ monkeypatch.setattr(persistence, "_load_page_paths_and_hotspots", load)+ state = _stale_state()++ persistence.stamp_file_page_scope("repo", state, [_parsed("a.py")], _Builder(), {})++ assert state == _stale_state()+++def test_no_parse_is_a_no_op(no_config) -> None:+ state = _stale_state()+ persistence.stamp_file_page_scope("repo", state, [], _Builder(), {})+ assert state == _stale_state()+++def test_near_clone_losers_are_not_eligible() -> None:+ clones = [_parsed(f"gen/c{i}.py", symbols=("run",)) for i in range(3)]+ pagerank = {"gen/c0.py": 1.0}+ assert eligible_file_page_paths(clones, pagerank, hotspots=set()) == {"gen/c0.py"}+ every = eligible_file_page_paths(clones, pagerank, hotspots=set(), dedupe_near_clones=False)+ assert len(every) == 3+++def test_counts_match_the_page_list_form() -> None:+ pages = [SimpleNamespace(page_type="file_page")] * 3 + [+ SimpleNamespace(page_type="symbol_spotlight")+ ]+ assert file_page_scope(configured_cap=None, eligible=5, generated_pages=pages) == (+ file_page_counts(configured_cap=None, eligible=5, generated=3)+ )+++async def test_store_read_skips_tombstones(tmp_path, monkeypatch) -> None:+ from repowise.core.persistence import (+ create_engine,+ create_session_factory,+ get_session,+ init_db,+ upsert_repository,+ )+ from repowise.core.persistence.crud import upsert_page+ from tests.unit.persistence.helpers import make_page_kwargs++ url = f"sqlite+aiosqlite:///{(tmp_path / 'wiki.db').as_posix()}"+ monkeypatch.setattr("repowise.cli.helpers.get_db_url_for_repo", lambda _repo: url)+ engine = create_engine(url)+ await init_db(engine)+ async with get_session(create_session_factory(engine)) as session:+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))+ for path, status in (("a.py", "fresh"), ("b.py", "stale"), ("c.py", "tombstone")):+ await upsert_page(+ session,+ **make_page_kwargs(+ repo.id, page_id=f"file_page:{path}", target_path=path, freshness_status=status+ ),+ )+ await engine.dispose()++ assert await persistence._load_page_paths_and_hotspots(tmp_path) == ({"a.py", "b.py"}, set())