Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(search): write full-text pages in fixed-size chunks (#2780)

Raghav Chamadiyaf95046b65b20
diff --git a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
index a622e9f65..173558d6a 100644
--- a/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
+++ b/packages/cli/src/repowise/cli/commands/doctor_cmd/repo_checks.py
@@ -805,15 +805,14 @@ def _run_repo_checks(
rows = await session.execute(
select(Page).where(Page.id.in_(list(missing_from_fts)))
)
- for page in rows.scalars().all():
- await fts.index(
- page.id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
- repaired += 1
+ # ORM rows key on ``id``, not ``page_id``, so they go
+ # to index_many as tuples rather than to index_pages.
+ batch = [
+ (p.id, p.title, p.content, p.summary, p.target_path)
+ for p in rows.scalars().all()
+ ]
+ await fts.index_many(batch)
+ repaired += len(batch)
# Repair vector store: re-embed missing pages, delete orphaned
lance_dir = repowise_dir / "lancedb"
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index ff7b61b31..59e673700 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -267,12 +267,7 @@ async def persist_result(
if fts is not None and swept_page_ids:
await fts.delete_many(swept_page_ids)
if fts is not None and result.generated_pages:
- await fts.index_many(
- [
- (page.page_id, page.title, page.content, page.summary, page.target_path)
- for page in result.generated_pages
- ]
- )
+ await fts.index_pages(result.generated_pages)
await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))
# Stamp the analysis (+ generation) phases in the resume ledger now that
diff --git a/packages/cli/src/repowise/cli/commands/restyle_cmd.py b/packages/cli/src/repowise/cli/commands/restyle_cmd.py
index 63f090e55..5c2b41ad6 100644
--- a/packages/cli/src/repowise/cli/commands/restyle_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/restyle_cmd.py
@@ -131,14 +131,7 @@ async def _run_restyle(
try:
fts = FullTextSearch(engine)
await fts.ensure_index()
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(generated_pages)
except Exception:
pass # FTS indexing is best-effort
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
index 7cf364964..00f506f58 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/deterministic.py
@@ -376,14 +376,7 @@ async def _persist_async(
try:
fts = FullTextSearch(engine)
await fts.ensure_index()
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(generated_pages)
except Exception as exc:
degraded.append(f"Full-text index: {exc}")
finally:
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 6851568e1..454fb7bc9 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -1338,14 +1338,7 @@ async def _persist_full_update_async(
try:
fts = FullTextSearch(engine)
await fts.ensure_index()
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(generated_pages)
# A tombstone can never be an answer — hydration drops it — but
# retrieval fetches a fixed number of rows before that check runs,
# so every tombstone left in the index costs a real candidate its
diff --git a/packages/cli/src/repowise/cli/commands/upgrade_flow.py b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
index a6767fb03..a68be0d8a 100644
--- a/packages/cli/src/repowise/cli/commands/upgrade_flow.py
+++ b/packages/cli/src/repowise/cli/commands/upgrade_flow.py
@@ -226,14 +226,7 @@ async def _sync_authoritative_fts(
await fts.ensure_index()
if cleanup_ids:
await fts.delete_many(sorted(cleanup_ids))
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(generated_pages)
except BaseException:
record_cleanup_debt(repo_path, "fts", cleanup_ids)
raise
diff --git a/packages/cli/src/repowise/cli/commands/workspace_cmd.py b/packages/cli/src/repowise/cli/commands/workspace_cmd.py
index 684a6da12..d8bb1728d 100644
--- a/packages/cli/src/repowise/cli/commands/workspace_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/workspace_cmd.py
@@ -803,14 +803,7 @@ def _generate_docs_for_added_repo(
await upsert_pages_from_generated(session, pages, repo.id)
fts = FullTextSearch(engine)
await fts.ensure_index()
- for p in pages:
- await fts.index(
- p.page_id,
- p.title,
- p.content,
- summary=p.summary,
- target_path=p.target_path,
- )
+ await fts.index_pages(pages)
await engine.dispose()
return len(pages)
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index 4b644d4c3..cb0d24d1d 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -18,7 +18,7 @@ from __future__ import annotations
import logging
from collections import OrderedDict
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from typing import Any, Literal
@@ -32,6 +32,7 @@ from repowise.core.fts_query import build_fts5_query as _build_fts5_query
from repowise.core.fts_query import match_term as _match_term
from repowise.core.fts_query import meaningful_terms as _meaningful_terms
+from .batches import chunked
from .information_floor import information_floor, meets_information_floor, substantive_text
@@ -451,6 +452,21 @@ class FullTextSearch:
"""
await self.index_many([(page_id, title, content, summary, target_path)])
+ async def index_pages(self, pages: Iterable[Any]) -> None:
+ """Index generated pages, one :meth:`index_many` transaction per chunk.
+
+ Each entry needs ``page_id``, ``title``, ``content``, ``summary`` and
+ ``target_path``. Use this, not :meth:`index` in a loop: a write deletes
+ by ``page_id``, which FTS5 stores unindexed, so every delete scans the
+ whole index. Per page that is quadratic in wiki size (18 GB of reads
+ for a 2,300-page rebuild); chunked it is one scan per chunk, while a
+ failure loses at most one chunk and memory holds one chunk's copies.
+ """
+ for chunk in chunked(list(pages)):
+ await self.index_many(
+ [(p.page_id, p.title, p.content, p.summary, p.target_path) for p in chunk]
+ )
+
async def index_many(
self,
pages: Sequence[tuple[str, str, str, str | None, str | None]],
diff --git a/packages/core/src/repowise/core/pipeline/scoped_generation.py b/packages/core/src/repowise/core/pipeline/scoped_generation.py
index 7faf53121..d149788b0 100644
--- a/packages/core/src/repowise/core/pipeline/scoped_generation.py
+++ b/packages/core/src/repowise/core/pipeline/scoped_generation.py
@@ -301,14 +301,7 @@ async def execute_scoped_generation(
try:
if swept_page_ids:
await fts.delete_many(swept_page_ids)
- for page in generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(generated_pages)
except Exception as exc:
logger.debug("fts_index_skipped", error=str(exc))
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index 45b53efb9..c17ad937a 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -667,14 +667,7 @@ async def _refresh_fts(fts: Any, swept_page_ids: Any, pages: list) -> None:
return
if swept_page_ids:
await fts.delete_many(swept_page_ids)
- for page in pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
+ await fts.index_pages(pages)
async def _record_index_completion(
diff --git a/tests/unit/cli/test_full_upgrade_progress.py b/tests/unit/cli/test_full_upgrade_progress.py
index 77e65bc0b..5ac3c699e 100644
--- a/tests/unit/cli/test_full_upgrade_progress.py
+++ b/tests/unit/cli/test_full_upgrade_progress.py
@@ -87,8 +87,8 @@ def test_full_upgrade_fts_cleanup_retries_after_failure(
raise RuntimeError("temporary FTS failure")
deleted.append(page_ids)
- async def index(self, page_id: str, *_args, **_kwargs) -> None:
- indexed.append(page_id)
+ async def index_pages(self, pages) -> None:
+ indexed.extend(page.page_id for page in pages)
import repowise.core.persistence.search as search
diff --git a/tests/unit/cli/test_update_fts_batched.py b/tests/unit/cli/test_update_fts_batched.py
new file mode 100644
index 000000000..e3733b835
--- /dev/null
+++ b/tests/unit/cli/test_update_fts_batched.py
@@ -0,0 +1,68 @@
+"""The update's page persist writes the full-text index in one batch.
+
+Every FTS write deletes by ``page_id``, which the index stores unindexed, so a
+per-page loop rescans the whole index once per page. On a large wiki that was
+the dominant cost of a full re-render, so the write goes in fixed-size chunks:
+one scan per chunk, and a failure or the memory held is bounded to one chunk.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+
+from repowise.cli.commands.update_cmd.deterministic import persist_deterministic_pages
+from repowise.core.generation.models import GeneratedPage
+from repowise.core.persistence.search import FullTextSearch
+
+
+def _page(i: int) -> GeneratedPage:
+ now = datetime.now(UTC).isoformat()
+ return GeneratedPage(
+ page_id=f"file_page:m{i}.py",
+ page_type="file_page",
+ title=f"File: m{i}.py",
+ content=f"Module {i} documents widget number {i}.",
+ source_hash=f"hash-{i}",
+ model_name="template",
+ provider_name="template",
+ input_tokens=0,
+ output_tokens=0,
+ cached_tokens=0,
+ generation_level=2,
+ target_path=f"m{i}.py",
+ created_at=now,
+ updated_at=now,
+ )
+
+
+def repo_dir(tmp_path, monkeypatch):
+ (tmp_path / ".repowise").mkdir()
+ db = tmp_path / ".repowise" / "wiki.db"
+ monkeypatch.setenv("REPOWISE_DB_URL", f"sqlite+aiosqlite:///{db.as_posix()}")
+ return tmp_path
+
+
+def test_persist_indexes_pages_in_fixed_size_chunks(repo_dir, monkeypatch):
+ calls: list[int] = []
+ original = FullTextSearch.index_many
+
+ async def _spy(self, pages):
+ calls.append(len(pages))
+ await original(self, pages)
+
+ monkeypatch.setattr(FullTextSearch, "index_many", _spy)
+
+ degraded: list[str] = []
+ total = persist_deterministic_pages(
+ repo_path=repo_dir,
+ generated_pages=[_page(i) for i in range(600)],
+ decay_paths=[],
+ degraded=degraded,
+ )
+
+ assert degraded == []
+ assert total == 600
+ assert calls == [500, 100]
diff --git a/tests/unit/cli/test_update_persist_reliability.py b/tests/unit/cli/test_update_persist_reliability.py
index 9e4524daf..ba265b6ec 100644
--- a/tests/unit/cli/test_update_persist_reliability.py
+++ b/tests/unit/cli/test_update_persist_reliability.py
@@ -166,7 +166,7 @@ def test_failed_fts_cleanup_debt_retries_on_next_persist(tmp_path: Path, monkeyp
async def ensure_index(self):
return None
- async def index(self, *args, **kwargs):
+ async def index_pages(self, pages):
return None
async def delete_many(self, page_ids):
diff --git a/tests/unit/cli/test_workspace_kg_artifact_wiring.py b/tests/unit/cli/test_workspace_kg_artifact_wiring.py
index c46b06978..413d91238 100644
--- a/tests/unit/cli/test_workspace_kg_artifact_wiring.py
+++ b/tests/unit/cli/test_workspace_kg_artifact_wiring.py
@@ -48,7 +48,7 @@ def test_generate_docs_for_added_repo_passes_repo_path(tmp_path):
fts = MagicMock()
fts.ensure_index = AsyncMock()
- fts.index = AsyncMock()
+ fts.index_pages = AsyncMock()
engine = MagicMock()
engine.dispose = AsyncMock()
diff --git a/tests/unit/persistence/test_search.py b/tests/unit/persistence/test_search.py
index 2d800f987..bdc2916c6 100644
--- a/tests/unit/persistence/test_search.py
+++ b/tests/unit/persistence/test_search.py
@@ -311,3 +311,47 @@ async def test_full_text_search_repository_id_filter(async_engine, async_session
b_hits = await fts.search("queue", repository_id="repo-b-id")
assert [r.page_id for r in b_hits] == ["p-b"]
+
+
+# ---------------------------------------------------------------------------
+# Batched page writes
+# ---------------------------------------------------------------------------
+
+
+async def test_index_pages_scans_the_index_once_per_id_chunk(async_engine, fts):
+ """A delete by the unindexed ``page_id`` scans the whole index, so a wiki
+ written page by page is quadratic. ``index_pages`` must issue one delete
+ per id chunk, not one per page, and leave the same rows the loop did."""
+ from types import SimpleNamespace
+
+ from sqlalchemy import event
+ from sqlalchemy.sql import text
+
+ pages = [
+ SimpleNamespace(
+ page_id=f"file_page:m{i}.py",
+ title=f"File: m{i}.py",
+ content=f"Module {i} documents widget number {i}.",
+ summary=f"summary {i}",
+ target_path=f"m{i}.py",
+ )
+ for i in range(1100)
+ ]
+ deletes: list[str] = []
+
+ def _count(conn, cursor, statement, *_):
+ if statement.startswith("DELETE FROM page_fts"):
+ deletes.append(statement)
+
+ event.listen(async_engine.sync_engine, "before_cursor_execute", _count)
+ try:
+ await fts.index_pages(pages)
+ finally:
+ event.remove(async_engine.sync_engine, "before_cursor_execute", _count)
+
+ assert len(deletes) == 3 # 1100 ids in chunks of 500
+ async with async_engine.connect() as conn:
+ rows = (
+ await conn.execute(text("SELECT page_id, title, summary, target_path FROM page_fts"))
+ ).fetchall()
+ assert sorted(rows) == sorted((p.page_id, p.title, p.summary, p.target_path) for p in pages)