Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(cli): stop capping the page listing at 10000 (#2061)

Azhar Dewjie94e42d66edd
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
index 428094368..3a867580e 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
@@ -60,6 +60,57 @@ def _is_stub_fallback_row(page) -> bool:
return isinstance(meta, dict) and STUB_FALLBACK_ERROR in meta
+
+async def _page_count(session: object, repo_id: str) -> int:
+ """Count this repository's pages in SQL.
+
+ Counted in the database rather than by measuring a fetched list. The
+ previous spelling took ``len()`` of a ``list_pages`` page, whose ``limit``
+ defaults to 100 and was passed 10000 here, so every repository with more
+ than 10000 pages reported exactly 10000 — a plausible-looking number that
+ never moves.
+ """
+ from sqlalchemy import func, select
+
+ from repowise.core.persistence.models import Page
+
+ result = await session.execute( # type: ignore[attr-defined]
+ select(func.count()).select_from(Page).where(Page.repository_id == repo_id)
+ )
+ return int(result.scalar_one())
+
+
+async def _all_pages_for_reconciliation(session: object, repo_id: str) -> list:
+ """Every page this repository has, for reconciling against the indexes.
+
+ The reconciliation asks which store entries have no page behind them, so
+ anything it cannot see it calls an orphan. It read the pages through
+ ``list_pages(limit=10000)`` — a paginated listing helper whose ``limit``
+ defaults to 100 — and treated that page-one result as the whole database.
+ Every page past the 10000th was therefore invisible, and each one turned
+ its vector and FTS row into a phantom orphan: on a 18900-page repository,
+ 8900 of them. ``--repair`` deletes what this reports, so what it removed
+ was the live index.
+
+ Three columns rather than whole rows: the id to match against the stores,
+ the content for the information floor, and metadata_json for the stub
+ predicate. Nothing downstream reads any other field, and hydrating full
+ ORM objects for every page only to discard them is what made a cap look
+ necessary in the first place.
+ """
+ from sqlalchemy import select
+
+ from repowise.core.persistence.models import Page
+
+ result = await session.execute( # type: ignore[attr-defined]
+ select(Page.id, Page.content, Page.metadata_json).where(
+ Page.repository_id == repo_id
+ )
+ )
+ return list(result.all())
+
+
+
def _run_repo_checks(
repo_path: _DoctorPath, repair: bool, *, fmt: str = "table"
) -> tuple[bool, list[DoctorCheck]]:
@@ -100,7 +151,6 @@ def _run_repo_checks(
create_session_factory,
get_repository_by_path,
get_session,
- list_pages,
)
url = get_db_url_for_repo(repo_path)
@@ -113,8 +163,7 @@ def _run_repo_checks(
async with get_session(sf) as session:
repo = await get_repository_by_path(session, str(repo_path))
if repo:
- pages = await list_pages(session, repo.id, limit=10000)
- count = len(pages)
+ count = await _page_count(session, repo.id)
await engine.dispose()
return count
@@ -264,7 +313,6 @@ def _run_repo_checks(
create_session_factory,
get_repository_by_path,
get_session,
- list_pages,
)
from repowise.core.persistence.information_floor import (
meets_information_floor,
@@ -285,7 +333,8 @@ def _run_repo_checks(
if not repo:
await engine.dispose()
return set(), set(), set(), set()
- pages = await list_pages(session, repo.id, limit=10000)
+ pages = await _all_pages_for_reconciliation(session, repo.id)
+ sql_ids = {p.id for p in pages}
# ``Page``'s primary key is the column ``id``; there is no
# ``page_id`` attribute. The old spelling raised here on
# every run, was swallowed below, and left both store
@@ -293,7 +342,7 @@ def _run_repo_checks(
# drift was ever detected and --repair never had anything
# to repair. The FTS repair block had the same defect and
# was fixed; this half was missed.
- sql_ids = {p.id for p in pages}
+
# The page vector store also holds decision embeddings under
# the "decision:<id>" namespace, so they belong on the SQL
# side of the ORPHAN check (but NOT FTS, which only indexes
diff --git a/packages/cli/src/repowise/cli/commands/export_cmd.py b/packages/cli/src/repowise/cli/commands/export_cmd.py
index 11e792061..fbd7ad272 100644
--- a/packages/cli/src/repowise/cli/commands/export_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/export_cmd.py
@@ -16,6 +16,11 @@ from repowise.cli.helpers import (
run_async,
)
+# Rows per page of the export walk. Large enough that a big wiki costs few
+# round trips, small enough that one batch is never the whole database in
+# memory at once.
+_EXPORT_PAGE_BATCH = 1000
+
@click.command("export")
@click.argument("path", required=False, default=None)
@@ -126,9 +131,24 @@ def export_command(
# A reader-facing export drops tombstones so it does not write a
# page for a file that no longer exists. --full is the archival
# mode, so it keeps them for a complete record.
- pages = await list_pages(
- session, repo.id, include_tombstones=full_export, limit=10000
- )
+ # Paged through rather than fetched with a raised cap. `list_pages`
+ # is the paginated listing helper whose limit defaults to 100; the
+ # single `limit=10000` call this replaces silently stopped an export
+ # at 10000 pages, so a larger wiki exported a truncated archive that
+ # looked complete. Export needs whole rows, so this walks the pages
+ # rather than narrowing the columns.
+ pages = []
+ while True:
+ batch = await list_pages(
+ session,
+ repo.id,
+ include_tombstones=full_export,
+ limit=_EXPORT_PAGE_BATCH,
+ offset=len(pages),
+ )
+ pages.extend(batch)
+ if len(batch) < _EXPORT_PAGE_BATCH:
+ break
if full_export and fmt == "json":
from sqlalchemy import select
diff --git a/packages/cli/src/repowise/cli/commands/status_cmd.py b/packages/cli/src/repowise/cli/commands/status_cmd.py
index 2d9c34287..43ad4ef58 100644
--- a/packages/cli/src/repowise/cli/commands/status_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/status_cmd.py
@@ -155,7 +155,6 @@ async def _query_pages(repo_path: Path) -> tuple[dict[str, int], int]:
create_session_factory,
get_repository_by_path,
get_session,
- list_pages,
)
url = get_db_url_for_repo(repo_path)
@@ -170,10 +169,32 @@ async def _query_pages(repo_path: Path) -> tuple[dict[str, int], int]:
repo = await get_repository_by_path(session, str(repo_path))
if repo is None:
return counts, total_tokens
- pages = await list_pages(session, repo.id, include_tombstones=False, limit=10000)
- for p in pages:
- counts[p.page_type] = counts.get(p.page_type, 0) + 1
- total_tokens += (p.input_tokens or 0) + (p.output_tokens or 0)
+ # Aggregated in SQL rather than by counting a fetched list. This
+ # read `list_pages(..., limit=10000)`, and `list_pages` is the
+ # paginated listing helper whose limit defaults to 100 — so the
+ # per-type table stopped at exactly 10000 pages and the token total
+ # with it, while the header above printed the true count from a
+ # different query. A wiki over the cap disagreed with itself.
+ from sqlalchemy import func, select
+
+ from repowise.core.persistence.models import Page
+
+ rows = await session.execute(
+ select(
+ Page.page_type,
+ func.count(),
+ func.sum(func.coalesce(Page.input_tokens, 0)),
+ func.sum(func.coalesce(Page.output_tokens, 0)),
+ )
+ .where(
+ Page.repository_id == repo.id,
+ Page.freshness_status != "tombstone",
+ )
+ .group_by(Page.page_type)
+ )
+ for page_type, count, in_tokens, out_tokens in rows.all():
+ counts[page_type] = counts.get(page_type, 0) + count
+ total_tokens += int(in_tokens or 0) + int(out_tokens or 0)
finally:
await engine.dispose()
return counts, total_tokens
diff --git a/tests/unit/cli/test_doctor_page_cap.py b/tests/unit/cli/test_doctor_page_cap.py
new file mode 100644
index 000000000..2266c612c
--- /dev/null
+++ b/tests/unit/cli/test_doctor_page_cap.py
@@ -0,0 +1,141 @@
+"""``doctor`` reconciles against every page, not the first ten thousand.
+
+The SQL side of both store reconciliations was read with
+``list_pages(session, repo.id, limit=10000)``. ``list_pages`` is the paginated
+listing helper whose ``limit`` defaults to 100; passing 10000 does not remove
+the cap, it raises it. Every page past the 10000th was invisible to the check,
+so its vector and FTS row had no page behind them and was counted as an orphan
+— on an 18900-page repository, 8900 phantom vector orphans and 3877 phantom
+FTS ones, a permanent FAIL that no amount of reindexing moved.
+
+``--repair`` deletes what this check reports. On any repository over the cap it
+would therefore delete the live index it was asked to repair.
+
+The same fetch backed the "Database" row, which reported exactly 10000 pages
+for every repository larger than that.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+
+from repowise.cli.commands.doctor_cmd import repo_checks
+
+# One page past the old cap: the last page is the one that must stay visible.
+PAGE_TOTAL = 10001
+ORPHAN_CANARY = "file_page:beyond_the_cap.py"
+
+
+async def _build_repo(tmp_path: Path) -> Path:
+ """A repository with more pages than the old fetch could see.
+
+ The canary is indexed in FTS and is the OLDEST page by ``updated_at``,
+ which is what ``list_pages`` sorts on, descending — so under the old cap
+ it was the first page dropped, and was reported as an orphan.
+ """
+ import datetime
+
+ import git as gitpython
+ from sqlalchemy import insert
+
+ from repowise.core.persistence import (
+ FullTextSearch,
+ create_engine,
+ create_session_factory,
+ get_session,
+ )
+ from repowise.core.persistence.crud import upsert_repository
+ from repowise.core.persistence.database import init_db
+ from repowise.core.persistence.models import Page
+
+ repo_path = (tmp_path / "repo").resolve()
+ repo_path.mkdir()
+ gitpython.Repo.init(repo_path)
+ repowise_dir = repo_path / ".repowise"
+ repowise_dir.mkdir()
+
+ engine = create_engine(f"sqlite+aiosqlite:///{repowise_dir / 'wiki.db'}")
+ await init_db(engine)
+ sf = create_session_factory(engine)
+ async with get_session(sf) as session:
+ repo = await upsert_repository(
+ session, name="repo", local_path=str(repo_path), url="https://example.test/repo"
+ )
+ # Bulk insert: the point is the row count, and 10001 upserts through the
+ # CRUD layer would make this test slow enough that nobody runs it.
+ base = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)
+
+ def _stamp(i: int) -> datetime.datetime:
+ return base + datetime.timedelta(seconds=i)
+
+ rows = [
+ {
+ "created_at": _stamp(i),
+ "updated_at": _stamp(i),
+ "id": f"file_page:f{i}.py",
+ "repository_id": repo.id,
+ "page_type": "file_page",
+ "title": f"File: f{i}.py",
+ "content": "Body with enough words to clear the information floor.",
+ "summary": "",
+ "target_path": f"f{i}.py",
+ "source_hash": "",
+ "model_name": "mock",
+ "provider_name": "mock",
+ "freshness_status": "fresh",
+ }
+ for i in range(PAGE_TOTAL - 1)
+ ]
+ rows.append(
+ {
+ "created_at": base - datetime.timedelta(days=1),
+ "updated_at": base - datetime.timedelta(days=1),
+ "id": ORPHAN_CANARY,
+ "repository_id": repo.id,
+ "page_type": "file_page",
+ "title": "File: beyond_the_cap.py",
+ "content": "Body with enough words to clear the information floor.",
+ "summary": "",
+ "target_path": "beyond_the_cap.py",
+ "source_hash": "",
+ "model_name": "mock",
+ "provider_name": "mock",
+ "freshness_status": "fresh",
+ }
+ )
+ await session.execute(insert(Page), rows)
+ await session.commit()
+
+ # Every page is indexed, so a healthy repository reports no drift in either
+ # direction. That isolates what this test is about: the canary's FTS row has
+ # a page behind it, and is only an "orphan" if the check cannot see it.
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ await fts.index_many(
+ [(r["id"], r["title"], r["content"], "", r["target_path"]) for r in rows]
+ )
+ await engine.dispose()
+ return repo_path
+
+
+def _rows(repo_path: Path) -> dict[str, tuple[bool, str]]:
+ _all_ok, checks = repo_checks._run_repo_checks(repo_path, repair=False)
+ return {c.name: (c.ok, c.detail) for c in checks}
+
+
+def test_a_page_past_the_old_cap_is_not_an_orphan(tmp_path: Path) -> None:
+ """The canary has a page behind it, so its FTS row is not drift."""
+ rows = _rows(asyncio.run(_build_repo(tmp_path)))
+
+ ok, detail = rows["SQL ↔ FTS Index"]
+ assert detail == "in sync"
+ assert ok is True
+
+
+def test_the_database_row_counts_every_page(tmp_path: Path) -> None:
+ """It reported exactly 10000 for any repository larger than that."""
+ rows = _rows(asyncio.run(_build_repo(tmp_path)))
+
+ _ok, detail = rows["Database"]
+ assert detail == f"{PAGE_TOTAL} pages"
diff --git a/tests/unit/cli/test_page_listing_is_not_capped.py b/tests/unit/cli/test_page_listing_is_not_capped.py
new file mode 100644
index 000000000..640a83643
--- /dev/null
+++ b/tests/unit/cli/test_page_listing_is_not_capped.py
@@ -0,0 +1,123 @@
+"""``status`` and ``export`` cover every page, not the first ten thousand.
+
+Both read the wiki through ``list_pages(..., limit=10000)``. ``list_pages`` is
+the paginated listing helper whose ``limit`` defaults to 100, so passing 10000
+raises the cap rather than removing it, and both commands silently stopped
+there:
+
+- ``status`` printed a correct total in its header, from a different query, and
+ a per-type table underneath it that summed to exactly 10000. The token total
+ was truncated with it.
+- ``export`` wrote an archive missing every page past the 10000th, with nothing
+ in the output to say so.
+
+The same fetch backed ``doctor``'s two store reconciliations, where it turned
+every page past the cap into a phantom orphan that ``--repair`` then deleted.
+That half is pinned in ``test_doctor_page_cap.py``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+from pathlib import Path
+
+from repowise.cli.commands import status_cmd
+
+PAGE_TOTAL = 10001
+
+
+async def _build_repo(tmp_path: Path) -> Path:
+ """A repository with one more page than the old cap could return."""
+ import git as gitpython
+ from sqlalchemy import insert
+
+ from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ )
+ from repowise.core.persistence.crud import upsert_repository
+ from repowise.core.persistence.database import init_db
+ from repowise.core.persistence.models import Page
+
+ repo_path = (tmp_path / "repo").resolve()
+ repo_path.mkdir()
+ gitpython.Repo.init(repo_path)
+ repowise_dir = repo_path / ".repowise"
+ repowise_dir.mkdir()
+
+ engine = create_engine(f"sqlite+aiosqlite:///{repowise_dir / 'wiki.db'}")
+ await init_db(engine)
+ sf = create_session_factory(engine)
+ async with get_session(sf) as session:
+ repo = await upsert_repository(
+ session, name="repo", local_path=str(repo_path), url="https://example.test/repo"
+ )
+ base = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC)
+ # Bulk insert: the point is the row count, and 10001 upserts through the
+ # CRUD layer would make this slow enough that nobody runs it.
+ await session.execute(
+ insert(Page),
+ [
+ {
+ "created_at": base + datetime.timedelta(seconds=i),
+ "updated_at": base + datetime.timedelta(seconds=i),
+ "id": f"file_page:f{i}.py",
+ "repository_id": repo.id,
+ "page_type": "file_page",
+ "title": f"File: f{i}.py",
+ "content": "Body.",
+ "summary": "",
+ "target_path": f"f{i}.py",
+ "source_hash": "",
+ "model_name": "mock",
+ "provider_name": "mock",
+ "freshness_status": "fresh",
+ "input_tokens": 1,
+ "output_tokens": 2,
+ }
+ for i in range(PAGE_TOTAL)
+ ],
+ )
+ await session.commit()
+ await engine.dispose()
+ return repo_path
+
+
+def test_status_counts_every_page_and_its_tokens(tmp_path: Path) -> None:
+ """It reported exactly 10000 file pages, and the tokens of only those."""
+ repo_path = asyncio.run(_build_repo(tmp_path))
+
+ counts, total_tokens = asyncio.run(status_cmd._query_pages(repo_path))
+
+ assert counts["file_page"] == PAGE_TOTAL
+ assert total_tokens == PAGE_TOTAL * 3
+
+
+def test_export_walks_past_the_old_cap(tmp_path: Path) -> None:
+ """A full export stopped at 10000 pages and said nothing about it."""
+ from click.testing import CliRunner
+
+ from repowise.cli.commands.export_cmd import export_command
+
+ repo_path = asyncio.run(_build_repo(tmp_path))
+ out_dir = tmp_path / "export"
+
+ result = CliRunner().invoke(
+ export_command,
+ [str(repo_path), "--format", "json", "--output", str(out_dir)],
+ )
+
+ assert result.exit_code == 0, result.output
+
+ written = list(out_dir.rglob("*.json"))
+ assert written, f"nothing exported: {result.output}"
+ if len(written) == 1:
+ import json
+
+ payload = json.loads(written[0].read_text())
+ pages = payload["pages"] if isinstance(payload, dict) else payload
+ assert len(pages) == PAGE_TOTAL
+ else:
+ assert len(written) == PAGE_TOTAL