Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(update): recount the file-page scope on every update (#4220)

Raghav Chamadiyae4e2bf46de2f
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/command.py b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
index c8917a8e3..d3cf236bc 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/command.py
@@ -78,6 +78,7 @@ from .persistence import (
parser_changed,
rescan_security_if_changed,
security_scanner_changed,
+ stamp_file_page_scope,
stamp_head_commit,
)
from .reporting import (
@@ -1761,6 +1762,11 @@ def _docs_update(opts: UpdateOptions, run: UpdateRun) -> UpdateOutcome:
with timed(run.timings, "security_rescan"):
rescan_security_if_changed(repo_path, run.state, run.exclude_patterns)
+ # Saved before the editor files quote it.
+ stamp_file_page_scope(
+ repo_path, run.state, run.parsed_files, run.graph_builder, run.persisted_git_meta
+ )
+ save_state(repo_path, run.state)
_finish_outcome(repo_path, opts.agents_md, run.degraded, timings=run.timings)
_save_docs_state(run, docs)
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index f85ad8a10..645011f25 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -482,6 +482,98 @@ def _stamp_scope(repo_path: Any, new_state: dict, failed_steps: list[str]) -> No
)
+def stamp_file_page_scope(
+ repo_path: Any,
+ state: dict,
+ parsed_files: list | None,
+ graph_builder: Any,
+ git_meta_map: dict | None,
+) -> None:
+ """Recount ``index_scope.file_pages`` from this run's parse and the stored pages.
+
+ Eligibility applies selection's gates except the cap, as init's figure does;
+ an unreadable store keeps the previous figures.
+ """
+ if not parsed_files or graph_builder is None:
+ return
+ from repowise.core.generation import GenerationConfig
+ from repowise.core.generation.selection.selector import eligible_file_page_paths
+ from repowise.core.index_scope import file_page_counts, resolve_index_scope, stamp_index_scope
+ from repowise.core.repo_config import load_repo_config
+
+ try:
+ pagerank = graph_builder.pagerank()
+ except Exception as exc: # a released graph cannot rank anything
+ log.debug("file_page_scope_skipped", error=str(exc))
+ return
+ stored = run_async(_load_page_paths_and_hotspots(repo_path))
+ if stored is None:
+ return
+ page_paths, hotspots = stored
+ for path, meta in (git_meta_map or {}).items():
+ if meta.get("is_hotspot"):
+ hotspots.add(path)
+ else:
+ hotspots.discard(path)
+ eligible = eligible_file_page_paths(
+ parsed_files,
+ pagerank,
+ hotspots=hotspots,
+ dedupe_near_clones=GenerationConfig.from_repo_config(
+ load_repo_config(Path(repo_path))
+ ).dedupe_near_clones,
+ )
+ config = load_config(repo_path)
+ stamp_index_scope(
+ state,
+ config,
+ file_pages=file_page_counts(
+ configured_cap=resolve_index_scope(state, config)["file_pages"]["configured_cap"],
+ eligible=len(eligible),
+ generated=len(eligible & page_paths),
+ ),
+ )
+
+
+async def _load_page_paths_and_hotspots(repo_path: Any) -> tuple[set[str], set[str]] | None:
+ """Paths with a live file page, and the stored hotspot paths."""
+ from sqlalchemy import select
+
+ from repowise.cli.helpers import get_db_url_for_repo
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_repository_by_path,
+ get_session,
+ )
+ from repowise.core.persistence.models import GitMetadata, Page
+
+ engine = create_engine(get_db_url_for_repo(repo_path))
+ try:
+ async with get_session(create_session_factory(engine)) as session:
+ repo = await get_repository_by_path(session, str(repo_path))
+ if repo is None:
+ return None
+ pages = await session.execute(
+ select(Page.target_path).where(
+ Page.repository_id == repo.id,
+ Page.page_type == "file_page",
+ Page.freshness_status != "tombstone",
+ )
+ )
+ hotspots = await session.execute(
+ select(GitMetadata.file_path).where(
+ GitMetadata.repository_id == repo.id, GitMetadata.is_hotspot.is_(True)
+ )
+ )
+ return set(pages.scalars()), set(hotspots.scalars())
+ except Exception as exc:
+ log.warning("file_page_scope_read_failed", error=str(exc))
+ return None
+ finally:
+ await engine.dispose()
+
+
def _persist_index_only_update(
repo_path: Any,
graph_builder: Any,
@@ -636,6 +728,7 @@ def _persist_index_only_update(
timings.stop("run")
new_state["phase_timings"] = timings.totals
_stamp_scope(repo_path, new_state, failed_steps)
+ stamp_file_page_scope(repo_path, new_state, parsed_files, graph_builder, git_meta_map)
save_state(repo_path, new_state)
elapsed = time.monotonic() - start
from .reporting import show_index_only_completion
diff --git a/packages/core/src/repowise/core/generation/selection/selector.py b/packages/core/src/repowise/core/generation/selection/selector.py
index 462080e24..2bb55a00c 100644
--- a/packages/core/src/repowise/core/generation/selection/selector.py
+++ b/packages/core/src/repowise/core/generation/selection/selector.py
@@ -305,6 +305,29 @@ def file_page_eligible(parsed: Any, *, is_hotspot: bool) -> bool:
)
+def eligible_file_page_paths(
+ parsed_files: list[Any],
+ pagerank: dict[str, float],
+ *,
+ hotspots: set[str],
+ dedupe_near_clones: bool = True,
+) -> set[str]:
+ """The files selection would consider for a page: every gate but the cap."""
+ from ..page_generator.helpers import _select_clone_representatives
+
+ drop = (
+ _select_clone_representatives([p for p in parsed_files if _is_code_file(p)], pagerank)
+ if dedupe_near_clones
+ else set()
+ )
+ return {
+ p.file_info.path
+ for p in parsed_files
+ if p.file_info.path not in drop
+ and file_page_eligible(p, is_hotspot=p.file_info.path in hotspots)
+ }
+
+
def _passes_importance_floor(parsed: Any) -> bool:
"""Whether *parsed* is worth a file page at all.
diff --git a/packages/core/src/repowise/core/index_scope.py b/packages/core/src/repowise/core/index_scope.py
index 7b65cd8ea..4310df701 100644
--- a/packages/core/src/repowise/core/index_scope.py
+++ b/packages/core/src/repowise/core/index_scope.py
@@ -430,9 +430,16 @@ def file_page_scope(
*, configured_cap: int | None, eligible: int, generated_pages: list[Any] | None
) -> dict[str, int | None]:
"""Build configured/effective/achieved file-page figures for one run."""
+ generated = sum(1 for page in (generated_pages or []) if page.page_type == "file_page")
+ return file_page_counts(configured_cap=configured_cap, eligible=eligible, generated=generated)
+
+
+def file_page_counts(
+ *, configured_cap: int | None, eligible: int, generated: int
+) -> dict[str, int | None]:
+ """The file-page figures from counts, for a run that holds no page list."""
from repowise.core.generation.selection import auto_file_page_cap
- generated = sum(1 for page in (generated_pages or []) if page.page_type == "file_page")
effective = configured_cap
if configured_cap is None:
effective = auto_file_page_cap(eligible)
diff --git a/tests/unit/cli/test_update_file_page_scope.py b/tests/unit/cli/test_update_file_page_scope.py
new file mode 100644
index 000000000..a9dd27137
--- /dev/null
+++ b/tests/unit/cli/test_update_file_page_scope.py
@@ -0,0 +1,168 @@
+"""`repowise update` recounts ``index_scope.file_pages`` (#3905)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from repowise.cli.commands.update_cmd import persistence
+from repowise.core.generation.selection.selector import eligible_file_page_paths
+from repowise.core.index_scope import file_page_counts, file_page_scope
+
+
+def _parsed(path: str, *, is_test: bool = False, symbols: tuple[str, ...] = ("f",)):
+ return SimpleNamespace(
+ file_info=SimpleNamespace(
+ path=path,
+ language="python",
+ size_bytes=5_000,
+ is_test=is_test,
+ is_api_contract=False,
+ is_entry_point=False,
+ ),
+ symbols=[SimpleNamespace(name=n, kind="function") for n in symbols],
+ )
+
+
+class _Builder:
+ def pagerank(self) -> dict[str, float]:
+ return {}
+
+
+def _stale_state() -> dict:
+ pages = {"configured_cap": None, "effective_cap": None, "eligible": 1, "generated": 1}
+ return {"index_scope": {"file_pages": {**pages, "omitted": 0}}}
+
+
+def no_config(monkeypatch):
+ monkeypatch.setattr(persistence, "load_config", lambda _repo: {})
+ monkeypatch.setattr("repowise.core.repo_config.load_repo_config", lambda _repo: {})
+
+
+def _stored(page_paths: set[str], hotspots: set[str]):
+ async def load(_repo):
+ return set(page_paths), set(hotspots)
+
+ return load
+
+
+def test_recounts_from_parse_and_stored_pages(monkeypatch, no_config) -> None:
+ # The test file's leftover page is not counted as generated.
+ monkeypatch.setattr(
+ persistence,
+ "_load_page_paths_and_hotspots",
+ _stored({"a.py", "b.py", "t.py"}, set()),
+ )
+ state = _stale_state()
+ parsed = [
+ _parsed("a.py", symbols=("fa",)),
+ _parsed("b.py", symbols=("fb",)),
+ _parsed("c.py", symbols=("fc",)),
+ _parsed("t.py", is_test=True),
+ ]
+
+ persistence.stamp_file_page_scope("repo", state, parsed, _Builder(), {})
+
+ pages = state["index_scope"]["file_pages"]
+ assert (pages["eligible"], pages["generated"], pages["omitted"]) == (3, 2, 1)
+
+
+def test_this_runs_hotspot_rows_win(monkeypatch, no_config) -> None:
+ monkeypatch.setattr(
+ persistence, "_load_page_paths_and_hotspots", _stored(set(), {"shim.py"})
+ )
+ state = _stale_state()
+ parsed = [_parsed("shim.py", symbols=()), _parsed("other.py", symbols=())]
+
+ rows = {"shim.py": {"is_hotspot": False}, "other.py": {"is_hotspot": True}}
+ persistence.stamp_file_page_scope("repo", state, parsed, _Builder(), rows)
+
+ pages = state["index_scope"]["file_pages"]
+ assert (pages["eligible"], pages["omitted"]) == (1, 1)
+
+
+def test_explicit_unlimited_cap_is_kept(monkeypatch, no_config) -> None:
+ monkeypatch.setattr(persistence, "_load_page_paths_and_hotspots", _stored({"a.py"}, set()))
+ state = _stale_state()
+ state["index_scope"]["file_pages"]["configured_cap"] = 0
+
+ persistence.stamp_file_page_scope("repo", state, [_parsed("a.py")], _Builder(), {})
+
+ pages = state["index_scope"]["file_pages"]
+ assert (pages["configured_cap"], pages["effective_cap"], pages["omitted"]) == (0, None, 0)
+
+
+def test_docs_update_stamps_before_the_editor_files_quote_it() -> None:
+ import inspect
+
+ from repowise.cli.commands.update_cmd.command import _docs_update
+
+ source = inspect.getsource(_docs_update)
+ assert source.index("stamp_file_page_scope") < source.index("_finish_outcome")
+
+
+def test_unreadable_store_keeps_previous_figures(monkeypatch, no_config) -> None:
+ async def load(_repo):
+ return None
+
+ monkeypatch.setattr(persistence, "_load_page_paths_and_hotspots", load)
+ state = _stale_state()
+
+ persistence.stamp_file_page_scope("repo", state, [_parsed("a.py")], _Builder(), {})
+
+ assert state == _stale_state()
+
+
+def test_no_parse_is_a_no_op(no_config) -> None:
+ state = _stale_state()
+ persistence.stamp_file_page_scope("repo", state, [], _Builder(), {})
+ assert state == _stale_state()
+
+
+def test_near_clone_losers_are_not_eligible() -> None:
+ clones = [_parsed(f"gen/c{i}.py", symbols=("run",)) for i in range(3)]
+ pagerank = {"gen/c0.py": 1.0}
+ assert eligible_file_page_paths(clones, pagerank, hotspots=set()) == {"gen/c0.py"}
+ every = eligible_file_page_paths(clones, pagerank, hotspots=set(), dedupe_near_clones=False)
+ assert len(every) == 3
+
+
+def test_counts_match_the_page_list_form() -> None:
+ pages = [SimpleNamespace(page_type="file_page")] * 3 + [
+ SimpleNamespace(page_type="symbol_spotlight")
+ ]
+ assert file_page_scope(configured_cap=None, eligible=5, generated_pages=pages) == (
+ file_page_counts(configured_cap=None, eligible=5, generated=3)
+ )
+
+
+async def test_store_read_skips_tombstones(tmp_path, monkeypatch) -> None:
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ init_db,
+ upsert_repository,
+ )
+ from repowise.core.persistence.crud import upsert_page
+ from tests.unit.persistence.helpers import make_page_kwargs
+
+ url = f"sqlite+aiosqlite:///{(tmp_path / 'wiki.db').as_posix()}"
+ monkeypatch.setattr("repowise.cli.helpers.get_db_url_for_repo", lambda _repo: url)
+ engine = create_engine(url)
+ await init_db(engine)
+ async with get_session(create_session_factory(engine)) as session:
+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))
+ for path, status in (("a.py", "fresh"), ("b.py", "stale"), ("c.py", "tombstone")):
+ await upsert_page(
+ session,
+ **make_page_kwargs(
+ repo.id, page_id=f"file_page:{path}", target_path=path, freshness_status=status
+ ),
+ )
+ await engine.dispose()
+
+ assert await persistence._load_page_paths_and_hotspots(tmp_path) == ({"a.py", "b.py"}, set())