Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(cli): stop capping the page listing at 10000 (#2061)
Azhar Dewjie94e42d66edd
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.pyindex 428094368..3a867580e 100644--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py@@ -60,6 +60,57 @@ def _is_stub_fallback_row(page) -> bool:return isinstance(meta, dict) and STUB_FALLBACK_ERROR in meta++async def _page_count(session: object, repo_id: str) -> int:+ """Count this repository's pages in SQL.++ Counted in the database rather than by measuring a fetched list. The+ previous spelling took ``len()`` of a ``list_pages`` page, whose ``limit``+ defaults to 100 and was passed 10000 here, so every repository with more+ than 10000 pages reported exactly 10000 — a plausible-looking number that+ never moves.+ """+ from sqlalchemy import func, select++ from repowise.core.persistence.models import Page++ result = await session.execute( # type: ignore[attr-defined]+ select(func.count()).select_from(Page).where(Page.repository_id == repo_id)+ )+ return int(result.scalar_one())+++async def _all_pages_for_reconciliation(session: object, repo_id: str) -> list:+ """Every page this repository has, for reconciling against the indexes.++ The reconciliation asks which store entries have no page behind them, so+ anything it cannot see it calls an orphan. It read the pages through+ ``list_pages(limit=10000)`` — a paginated listing helper whose ``limit``+ defaults to 100 — and treated that page-one result as the whole database.+ Every page past the 10000th was therefore invisible, and each one turned+ its vector and FTS row into a phantom orphan: on a 18900-page repository,+ 8900 of them. ``--repair`` deletes what this reports, so what it removed+ was the live index.++ Three columns rather than whole rows: the id to match against the stores,+ the content for the information floor, and metadata_json for the stub+ predicate. Nothing downstream reads any other field, and hydrating full+ ORM objects for every page only to discard them is what made a cap look+ necessary in the first place.+ """+ from sqlalchemy import select++ from repowise.core.persistence.models import Page++ result = await session.execute( # type: ignore[attr-defined]+ select(Page.id, Page.content, Page.metadata_json).where(+ Page.repository_id == repo_id+ )+ )+ return list(result.all())+++def _run_repo_checks(repo_path: _DoctorPath, repair: bool, *, fmt: str = "table") -> tuple[bool, list[DoctorCheck]]:@@ -100,7 +151,6 @@ def _run_repo_checks(create_session_factory,get_repository_by_path,get_session,- list_pages,)url = get_db_url_for_repo(repo_path)@@ -113,8 +163,7 @@ def _run_repo_checks(async with get_session(sf) as session:repo = await get_repository_by_path(session, str(repo_path))if repo:- pages = await list_pages(session, repo.id, limit=10000)- count = len(pages)+ count = await _page_count(session, repo.id)await engine.dispose()return count@@ -264,7 +313,6 @@ def _run_repo_checks(create_session_factory,get_repository_by_path,get_session,- list_pages,)from repowise.core.persistence.information_floor import (meets_information_floor,@@ -285,7 +333,8 @@ def _run_repo_checks(if not repo:await engine.dispose()return set(), set(), set(), set()- pages = await list_pages(session, repo.id, limit=10000)+ pages = await _all_pages_for_reconciliation(session, repo.id)+ sql_ids = {p.id for p in pages}# ``Page``'s primary key is the column ``id``; there is no# ``page_id`` attribute. The old spelling raised here on# every run, was swallowed below, and left both store@@ -293,7 +342,7 @@ def _run_repo_checks(# drift was ever detected and --repair never had anything# to repair. The FTS repair block had the same defect and# was fixed; this half was missed.- sql_ids = {p.id for p in pages}+# The page vector store also holds decision embeddings under# the "decision:<id>" namespace, so they belong on the SQL# side of the ORPHAN check (but NOT FTS, which only indexesdiff --git a/packages/cli/src/repowise/cli/commands/export_cmd.py b/packages/cli/src/repowise/cli/commands/export_cmd.pyindex 11e792061..fbd7ad272 100644--- a/packages/cli/src/repowise/cli/commands/export_cmd.py+++ b/packages/cli/src/repowise/cli/commands/export_cmd.py@@ -16,6 +16,11 @@ from repowise.cli.helpers import (run_async,)+# Rows per page of the export walk. Large enough that a big wiki costs few+# round trips, small enough that one batch is never the whole database in+# memory at once.+_EXPORT_PAGE_BATCH = 1000+@click.command("export")@click.argument("path", required=False, default=None)@@ -126,9 +131,24 @@ def export_command(# A reader-facing export drops tombstones so it does not write a# page for a file that no longer exists. --full is the archival# mode, so it keeps them for a complete record.- pages = await list_pages(- session, repo.id, include_tombstones=full_export, limit=10000- )+ # Paged through rather than fetched with a raised cap. `list_pages`+ # is the paginated listing helper whose limit defaults to 100; the+ # single `limit=10000` call this replaces silently stopped an export+ # at 10000 pages, so a larger wiki exported a truncated archive that+ # looked complete. Export needs whole rows, so this walks the pages+ # rather than narrowing the columns.+ pages = []+ while True:+ batch = await list_pages(+ session,+ repo.id,+ include_tombstones=full_export,+ limit=_EXPORT_PAGE_BATCH,+ offset=len(pages),+ )+ pages.extend(batch)+ if len(batch) < _EXPORT_PAGE_BATCH:+ breakif full_export and fmt == "json":from sqlalchemy import selectdiff --git a/packages/cli/src/repowise/cli/commands/status_cmd.py b/packages/cli/src/repowise/cli/commands/status_cmd.pyindex 2d9c34287..43ad4ef58 100644--- a/packages/cli/src/repowise/cli/commands/status_cmd.py+++ b/packages/cli/src/repowise/cli/commands/status_cmd.py@@ -155,7 +155,6 @@ async def _query_pages(repo_path: Path) -> tuple[dict[str, int], int]:create_session_factory,get_repository_by_path,get_session,- list_pages,)url = get_db_url_for_repo(repo_path)@@ -170,10 +169,32 @@ async def _query_pages(repo_path: Path) -> tuple[dict[str, int], int]:repo = await get_repository_by_path(session, str(repo_path))if repo is None:return counts, total_tokens- pages = await list_pages(session, repo.id, include_tombstones=False, limit=10000)- for p in pages:- counts[p.page_type] = counts.get(p.page_type, 0) + 1- total_tokens += (p.input_tokens or 0) + (p.output_tokens or 0)+ # Aggregated in SQL rather than by counting a fetched list. This+ # read `list_pages(..., limit=10000)`, and `list_pages` is the+ # paginated listing helper whose limit defaults to 100 — so the+ # per-type table stopped at exactly 10000 pages and the token total+ # with it, while the header above printed the true count from a+ # different query. A wiki over the cap disagreed with itself.+ from sqlalchemy import func, select++ from repowise.core.persistence.models import Page++ rows = await session.execute(+ select(+ Page.page_type,+ func.count(),+ func.sum(func.coalesce(Page.input_tokens, 0)),+ func.sum(func.coalesce(Page.output_tokens, 0)),+ )+ .where(+ Page.repository_id == repo.id,+ Page.freshness_status != "tombstone",+ )+ .group_by(Page.page_type)+ )+ for page_type, count, in_tokens, out_tokens in rows.all():+ counts[page_type] = counts.get(page_type, 0) + count+ total_tokens += int(in_tokens or 0) + int(out_tokens or 0)finally:await engine.dispose()return counts, total_tokensdiff --git a/tests/unit/cli/test_doctor_page_cap.py b/tests/unit/cli/test_doctor_page_cap.pynew file mode 100644index 000000000..2266c612c--- /dev/null+++ b/tests/unit/cli/test_doctor_page_cap.py@@ -0,0 +1,141 @@+"""``doctor`` reconciles against every page, not the first ten thousand.++The SQL side of both store reconciliations was read with+``list_pages(session, repo.id, limit=10000)``. ``list_pages`` is the paginated+listing helper whose ``limit`` defaults to 100; passing 10000 does not remove+the cap, it raises it. Every page past the 10000th was invisible to the check,+so its vector and FTS row had no page behind them and was counted as an orphan+— on an 18900-page repository, 8900 phantom vector orphans and 3877 phantom+FTS ones, a permanent FAIL that no amount of reindexing moved.++``--repair`` deletes what this check reports. On any repository over the cap it+would therefore delete the live index it was asked to repair.++The same fetch backed the "Database" row, which reported exactly 10000 pages+for every repository larger than that.+"""++from __future__ import annotations++import asyncio+from pathlib import Path++from repowise.cli.commands.doctor_cmd import repo_checks++# One page past the old cap: the last page is the one that must stay visible.+PAGE_TOTAL = 10001+ORPHAN_CANARY = "file_page:beyond_the_cap.py"+++async def _build_repo(tmp_path: Path) -> Path:+ """A repository with more pages than the old fetch could see.++ The canary is indexed in FTS and is the OLDEST page by ``updated_at``,+ which is what ``list_pages`` sorts on, descending — so under the old cap+ it was the first page dropped, and was reported as an orphan.+ """+ import datetime++ import git as gitpython+ from sqlalchemy import insert++ from repowise.core.persistence import (+ FullTextSearch,+ create_engine,+ create_session_factory,+ get_session,+ )+ from repowise.core.persistence.crud import upsert_repository+ from repowise.core.persistence.database import init_db+ from repowise.core.persistence.models import Page++ repo_path = (tmp_path / "repo").resolve()+ repo_path.mkdir()+ gitpython.Repo.init(repo_path)+ repowise_dir = repo_path / ".repowise"+ repowise_dir.mkdir()++ engine = create_engine(f"sqlite+aiosqlite:///{repowise_dir / 'wiki.db'}")+ await init_db(engine)+ sf = create_session_factory(engine)+ async with get_session(sf) as session:+ repo = await upsert_repository(+ session, name="repo", local_path=str(repo_path), url="https://example.test/repo"+ )+ # Bulk insert: the point is the row count, and 10001 upserts through the+ # CRUD layer would make this test slow enough that nobody runs it.+ base = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)++ def _stamp(i: int) -> datetime.datetime:+ return base + datetime.timedelta(seconds=i)++ rows = [+ {+ "created_at": _stamp(i),+ "updated_at": _stamp(i),+ "id": f"file_page:f{i}.py",+ "repository_id": repo.id,+ "page_type": "file_page",+ "title": f"File: f{i}.py",+ "content": "Body with enough words to clear the information floor.",+ "summary": "",+ "target_path": f"f{i}.py",+ "source_hash": "",+ "model_name": "mock",+ "provider_name": "mock",+ "freshness_status": "fresh",+ }+ for i in range(PAGE_TOTAL - 1)+ ]+ rows.append(+ {+ "created_at": base - datetime.timedelta(days=1),+ "updated_at": base - datetime.timedelta(days=1),+ "id": ORPHAN_CANARY,+ "repository_id": repo.id,+ "page_type": "file_page",+ "title": "File: beyond_the_cap.py",+ "content": "Body with enough words to clear the information floor.",+ "summary": "",+ "target_path": "beyond_the_cap.py",+ "source_hash": "",+ "model_name": "mock",+ "provider_name": "mock",+ "freshness_status": "fresh",+ }+ )+ await session.execute(insert(Page), rows)+ await session.commit()++ # Every page is indexed, so a healthy repository reports no drift in either+ # direction. That isolates what this test is about: the canary's FTS row has+ # a page behind it, and is only an "orphan" if the check cannot see it.+ fts = FullTextSearch(engine)+ await fts.ensure_index()+ await fts.index_many(+ [(r["id"], r["title"], r["content"], "", r["target_path"]) for r in rows]+ )+ await engine.dispose()+ return repo_path+++def _rows(repo_path: Path) -> dict[str, tuple[bool, str]]:+ _all_ok, checks = repo_checks._run_repo_checks(repo_path, repair=False)+ return {c.name: (c.ok, c.detail) for c in checks}+++def test_a_page_past_the_old_cap_is_not_an_orphan(tmp_path: Path) -> None:+ """The canary has a page behind it, so its FTS row is not drift."""+ rows = _rows(asyncio.run(_build_repo(tmp_path)))++ ok, detail = rows["SQL ↔ FTS Index"]+ assert detail == "in sync"+ assert ok is True+++def test_the_database_row_counts_every_page(tmp_path: Path) -> None:+ """It reported exactly 10000 for any repository larger than that."""+ rows = _rows(asyncio.run(_build_repo(tmp_path)))++ _ok, detail = rows["Database"]+ assert detail == f"{PAGE_TOTAL} pages"diff --git a/tests/unit/cli/test_page_listing_is_not_capped.py b/tests/unit/cli/test_page_listing_is_not_capped.pynew file mode 100644index 000000000..640a83643--- /dev/null+++ b/tests/unit/cli/test_page_listing_is_not_capped.py@@ -0,0 +1,123 @@+"""``status`` and ``export`` cover every page, not the first ten thousand.++Both read the wiki through ``list_pages(..., limit=10000)``. ``list_pages`` is+the paginated listing helper whose ``limit`` defaults to 100, so passing 10000+raises the cap rather than removing it, and both commands silently stopped+there:++- ``status`` printed a correct total in its header, from a different query, and+ a per-type table underneath it that summed to exactly 10000. The token total+ was truncated with it.+- ``export`` wrote an archive missing every page past the 10000th, with nothing+ in the output to say so.++The same fetch backed ``doctor``'s two store reconciliations, where it turned+every page past the cap into a phantom orphan that ``--repair`` then deleted.+That half is pinned in ``test_doctor_page_cap.py``.+"""++from __future__ import annotations++import asyncio+import datetime+from pathlib import Path++from repowise.cli.commands import status_cmd++PAGE_TOTAL = 10001+++async def _build_repo(tmp_path: Path) -> Path:+ """A repository with one more page than the old cap could return."""+ import git as gitpython+ from sqlalchemy import insert++ from repowise.core.persistence import (+ create_engine,+ create_session_factory,+ get_session,+ )+ from repowise.core.persistence.crud import upsert_repository+ from repowise.core.persistence.database import init_db+ from repowise.core.persistence.models import Page++ repo_path = (tmp_path / "repo").resolve()+ repo_path.mkdir()+ gitpython.Repo.init(repo_path)+ repowise_dir = repo_path / ".repowise"+ repowise_dir.mkdir()++ engine = create_engine(f"sqlite+aiosqlite:///{repowise_dir / 'wiki.db'}")+ await init_db(engine)+ sf = create_session_factory(engine)+ async with get_session(sf) as session:+ repo = await upsert_repository(+ session, name="repo", local_path=str(repo_path), url="https://example.test/repo"+ )+ base = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)+ # Bulk insert: the point is the row count, and 10001 upserts through the+ # CRUD layer would make this slow enough that nobody runs it.+ await session.execute(+ insert(Page),+ [+ {+ "created_at": base + datetime.timedelta(seconds=i),+ "updated_at": base + datetime.timedelta(seconds=i),+ "id": f"file_page:f{i}.py",+ "repository_id": repo.id,+ "page_type": "file_page",+ "title": f"File: f{i}.py",+ "content": "Body.",+ "summary": "",+ "target_path": f"f{i}.py",+ "source_hash": "",+ "model_name": "mock",+ "provider_name": "mock",+ "freshness_status": "fresh",+ "input_tokens": 1,+ "output_tokens": 2,+ }+ for i in range(PAGE_TOTAL)+ ],+ )+ await session.commit()+ await engine.dispose()+ return repo_path+++def test_status_counts_every_page_and_its_tokens(tmp_path: Path) -> None:+ """It reported exactly 10000 file pages, and the tokens of only those."""+ repo_path = asyncio.run(_build_repo(tmp_path))++ counts, total_tokens = asyncio.run(status_cmd._query_pages(repo_path))++ assert counts["file_page"] == PAGE_TOTAL+ assert total_tokens == PAGE_TOTAL * 3+++def test_export_walks_past_the_old_cap(tmp_path: Path) -> None:+ """A full export stopped at 10000 pages and said nothing about it."""+ from click.testing import CliRunner++ from repowise.cli.commands.export_cmd import export_command++ repo_path = asyncio.run(_build_repo(tmp_path))+ out_dir = tmp_path / "export"++ result = CliRunner().invoke(+ export_command,+ [str(repo_path), "--format", "json", "--output", str(out_dir)],+ )++ assert result.exit_code == 0, result.output++ written = list(out_dir.rglob("*.json"))+ assert written, f"nothing exported: {result.output}"+ if len(written) == 1:+ import json++ payload = json.loads(written[0].read_text())+ pages = payload["pages"] if isinstance(payload, dict) else payload+ assert len(pages) == PAGE_TOTAL+ else:+ assert len(written) == PAGE_TOTAL