Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(search): write full-text pages in fixed-size chunks (#2780)
Raghav Chamadiyaf95046b65b20
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.pyindex a622e9f65..173558d6a 100644--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py@@ -805,15 +805,14 @@ def _run_repo_checks(rows = await session.execute(select(Page).where(Page.id.in_(list(missing_from_fts))))- for page in rows.scalars().all():- await fts.index(- page.id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )- repaired += 1+ # ORM rows key on ``id``, not ``page_id``, so they go+ # to index_many as tuples rather than to index_pages.+ batch = [+ (p.id, p.title, p.content, p.summary, p.target_path)+ for p in rows.scalars().all()+ ]+ await fts.index_many(batch)+ repaired += len(batch)# Repair vector store: re-embed missing pages, delete orphanedlance_dir = repowise_dir / "lancedb"diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex ff7b61b31..59e673700 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -267,12 +267,7 @@ async def persist_result(if fts is not None and swept_page_ids:await fts.delete_many(swept_page_ids)if fts is not None and result.generated_pages:- await fts.index_many(- [- (page.page_id, page.title, page.content, page.summary, page.target_path)- for page in result.generated_pages- ]- )+ await fts.index_pages(result.generated_pages)await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))# Stamp the analysis (+ generation) phases in the resume ledger now thatdiff --git a/packages/cli/src/repowise/cli/commands/restyle_cmd.py b/packages/cli/src/repowise/cli/commands/restyle_cmd.pyindex 63f090e55..5c2b41ad6 100644--- a/packages/cli/src/repowise/cli/commands/restyle_cmd.py+++ b/packages/cli/src/repowise/cli/commands/restyle_cmd.py@@ -131,14 +131,7 @@ async def _run_restyle(try:fts = FullTextSearch(engine)await fts.ensure_index()- for page in generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(generated_pages)except Exception:pass # FTS indexing is best-effortdiff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.pyindex 7cf364964..00f506f58 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py@@ -376,14 +376,7 @@ async def _persist_async(try:fts = FullTextSearch(engine)await fts.ensure_index()- for page in generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(generated_pages)except Exception as exc:degraded.append(f"Full-text index: {exc}")finally:diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex 6851568e1..454fb7bc9 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -1338,14 +1338,7 @@ async def _persist_full_update_async(try:fts = FullTextSearch(engine)await fts.ensure_index()- for page in generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(generated_pages)# A tombstone can never be an answer — hydration drops it — but# retrieval fetches a fixed number of rows before that check runs,# so every tombstone left in the index costs a real candidate itsdiff --git a/packages/cli/src/repowise/cli/commands/upgrade_flow.py b/packages/cli/src/repowise/cli/commands/upgrade_flow.pyindex a6767fb03..a68be0d8a 100644--- a/packages/cli/src/repowise/cli/commands/upgrade_flow.py+++ b/packages/cli/src/repowise/cli/commands/upgrade_flow.py@@ -226,14 +226,7 @@ async def _sync_authoritative_fts(await fts.ensure_index()if cleanup_ids:await fts.delete_many(sorted(cleanup_ids))- for page in generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(generated_pages)except BaseException:record_cleanup_debt(repo_path, "fts", cleanup_ids)raisediff --git a/packages/cli/src/repowise/cli/commands/workspace_cmd.py b/packages/cli/src/repowise/cli/commands/workspace_cmd.pyindex 684a6da12..d8bb1728d 100644--- a/packages/cli/src/repowise/cli/commands/workspace_cmd.py+++ b/packages/cli/src/repowise/cli/commands/workspace_cmd.py@@ -803,14 +803,7 @@ def _generate_docs_for_added_repo(await upsert_pages_from_generated(session, pages, repo.id)fts = FullTextSearch(engine)await fts.ensure_index()- for p in pages:- await fts.index(- p.page_id,- p.title,- p.content,- summary=p.summary,- target_path=p.target_path,- )+ await fts.index_pages(pages)await engine.dispose()return len(pages)diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex 4b644d4c3..cb0d24d1d 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -18,7 +18,7 @@ from __future__ import annotationsimport loggingfrom collections import OrderedDict-from collections.abc import Sequence+from collections.abc import Iterable, Sequencefrom dataclasses import dataclassfrom typing import Any, Literal@@ -32,6 +32,7 @@ from repowise.core.fts_query import build_fts5_query as _build_fts5_queryfrom repowise.core.fts_query import match_term as _match_termfrom repowise.core.fts_query import meaningful_terms as _meaningful_terms+from .batches import chunkedfrom .information_floor import information_floor, meets_information_floor, substantive_text@@ -451,6 +452,21 @@ class FullTextSearch:"""await self.index_many([(page_id, title, content, summary, target_path)])+ async def index_pages(self, pages: Iterable[Any]) -> None:+ """Index generated pages, one :meth:`index_many` transaction per chunk.++ Each entry needs ``page_id``, ``title``, ``content``, ``summary`` and+ ``target_path``. Use this, not :meth:`index` in a loop: a write deletes+ by ``page_id``, which FTS5 stores unindexed, so every delete scans the+ whole index. Per page that is quadratic in wiki size (18 GB of reads+ for a 2,300-page rebuild); chunked it is one scan per chunk, while a+ failure loses at most one chunk and memory holds one chunk's copies.+ """+ for chunk in chunked(list(pages)):+ await self.index_many(+ [(p.page_id, p.title, p.content, p.summary, p.target_path) for p in chunk]+ )+async def index_many(self,pages: Sequence[tuple[str, str, str, str | None, str | None]],diff --git a/packages/core/src/repowise/core/pipeline/scoped_generation.py b/packages/core/src/repowise/core/pipeline/scoped_generation.pyindex 7faf53121..d149788b0 100644--- a/packages/core/src/repowise/core/pipeline/scoped_generation.py+++ b/packages/core/src/repowise/core/pipeline/scoped_generation.py@@ -301,14 +301,7 @@ async def execute_scoped_generation(try:if swept_page_ids:await fts.delete_many(swept_page_ids)- for page in generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(generated_pages)except Exception as exc:logger.debug("fts_index_skipped", error=str(exc))diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.pyindex 45b53efb9..c17ad937a 100644--- a/packages/server/src/repowise/server/job_executor.py+++ b/packages/server/src/repowise/server/job_executor.py@@ -667,14 +667,7 @@ async def _refresh_fts(fts: Any, swept_page_ids: Any, pages: list) -> None:returnif swept_page_ids:await fts.delete_many(swept_page_ids)- for page in pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )+ await fts.index_pages(pages)async def _record_index_completion(diff --git a/tests/unit/cli/test_full_upgrade_progress.py b/tests/unit/cli/test_full_upgrade_progress.pyindex 77e65bc0b..5ac3c699e 100644--- a/tests/unit/cli/test_full_upgrade_progress.py+++ b/tests/unit/cli/test_full_upgrade_progress.py@@ -87,8 +87,8 @@ def test_full_upgrade_fts_cleanup_retries_after_failure(raise RuntimeError("temporary FTS failure")deleted.append(page_ids)- async def index(self, page_id: str, *_args, **_kwargs) -> None:- indexed.append(page_id)+ async def index_pages(self, pages) -> None:+ indexed.extend(page.page_id for page in pages)import repowise.core.persistence.search as searchdiff --git a/tests/unit/cli/test_update_fts_batched.py b/tests/unit/cli/test_update_fts_batched.pynew file mode 100644index 000000000..e3733b835--- /dev/null+++ b/tests/unit/cli/test_update_fts_batched.py@@ -0,0 +1,68 @@+"""The update's page persist writes the full-text index in one batch.++Every FTS write deletes by ``page_id``, which the index stores unindexed, so a+per-page loop rescans the whole index once per page. On a large wiki that was+the dominant cost of a full re-render, so the write goes in fixed-size chunks:+one scan per chunk, and a failure or the memory held is bounded to one chunk.+"""++from __future__ import annotations++from datetime import UTC, datetime++import pytest++from repowise.cli.commands.update_cmd.deterministic import persist_deterministic_pages+from repowise.core.generation.models import GeneratedPage+from repowise.core.persistence.search import FullTextSearch+++def _page(i: int) -> GeneratedPage:+ now = datetime.now(UTC).isoformat()+ return GeneratedPage(+ page_id=f"file_page:m{i}.py",+ page_type="file_page",+ title=f"File: m{i}.py",+ content=f"Module {i} documents widget number {i}.",+ source_hash=f"hash-{i}",+ model_name="template",+ provider_name="template",+ input_tokens=0,+ output_tokens=0,+ cached_tokens=0,+ generation_level=2,+ target_path=f"m{i}.py",+ created_at=now,+ updated_at=now,+ )+++def repo_dir(tmp_path, monkeypatch):+ (tmp_path / ".repowise").mkdir()+ db = tmp_path / ".repowise" / "wiki.db"+ monkeypatch.setenv("REPOWISE_DB_URL", f"sqlite+aiosqlite:///{db.as_posix()}")+ return tmp_path+++def test_persist_indexes_pages_in_fixed_size_chunks(repo_dir, monkeypatch):+ calls: list[int] = []+ original = FullTextSearch.index_many++ async def _spy(self, pages):+ calls.append(len(pages))+ await original(self, pages)++ monkeypatch.setattr(FullTextSearch, "index_many", _spy)++ degraded: list[str] = []+ total = persist_deterministic_pages(+ repo_path=repo_dir,+ generated_pages=[_page(i) for i in range(600)],+ decay_paths=[],+ degraded=degraded,+ )++ assert degraded == []+ assert total == 600+ assert calls == [500, 100]diff --git a/tests/unit/cli/test_update_persist_reliability.py b/tests/unit/cli/test_update_persist_reliability.pyindex 9e4524daf..ba265b6ec 100644--- a/tests/unit/cli/test_update_persist_reliability.py+++ b/tests/unit/cli/test_update_persist_reliability.py@@ -166,7 +166,7 @@ def test_failed_fts_cleanup_debt_retries_on_next_persist(tmp_path: Path, monkeypasync def ensure_index(self):return None- async def index(self, *args, **kwargs):+ async def index_pages(self, pages):return Noneasync def delete_many(self, page_ids):diff --git a/tests/unit/cli/test_workspace_kg_artifact_wiring.py b/tests/unit/cli/test_workspace_kg_artifact_wiring.pyindex c46b06978..413d91238 100644--- a/tests/unit/cli/test_workspace_kg_artifact_wiring.py+++ b/tests/unit/cli/test_workspace_kg_artifact_wiring.py@@ -48,7 +48,7 @@ def test_generate_docs_for_added_repo_passes_repo_path(tmp_path):fts = MagicMock()fts.ensure_index = AsyncMock()- fts.index = AsyncMock()+ fts.index_pages = AsyncMock()engine = MagicMock()engine.dispose = AsyncMock()diff --git a/tests/unit/persistence/test_search.py b/tests/unit/persistence/test_search.pyindex 2d800f987..bdc2916c6 100644--- a/tests/unit/persistence/test_search.py+++ b/tests/unit/persistence/test_search.py@@ -311,3 +311,47 @@ async def test_full_text_search_repository_id_filter(async_engine, async_sessionb_hits = await fts.search("queue", repository_id="repo-b-id")assert [r.page_id for r in b_hits] == ["p-b"]+++# ---------------------------------------------------------------------------+# Batched page writes+# ---------------------------------------------------------------------------+++async def test_index_pages_scans_the_index_once_per_id_chunk(async_engine, fts):+ """A delete by the unindexed ``page_id`` scans the whole index, so a wiki+ written page by page is quadratic. ``index_pages`` must issue one delete+ per id chunk, not one per page, and leave the same rows the loop did."""+ from types import SimpleNamespace++ from sqlalchemy import event+ from sqlalchemy.sql import text++ pages = [+ SimpleNamespace(+ page_id=f"file_page:m{i}.py",+ title=f"File: m{i}.py",+ content=f"Module {i} documents widget number {i}.",+ summary=f"summary {i}",+ target_path=f"m{i}.py",+ )+ for i in range(1100)+ ]+ deletes: list[str] = []++ def _count(conn, cursor, statement, *_):+ if statement.startswith("DELETE FROM page_fts"):+ deletes.append(statement)++ event.listen(async_engine.sync_engine, "before_cursor_execute", _count)+ try:+ await fts.index_pages(pages)+ finally:+ event.remove(async_engine.sync_engine, "before_cursor_execute", _count)++ assert len(deletes) == 3 # 1100 ids in chunks of 500+ async with async_engine.connect() as conn:+ rows = (+ await conn.execute(text("SELECT page_id, title, summary, target_path FROM page_fts"))+ ).fetchall()+ assert sorted(rows) == sorted((p.page_id, p.title, p.summary, p.target_path) for p in pages)