Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(persist): write the full-text index in one transaction (#2052)

Raghav Chamadiya07abb3cba727
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
index 0fe07132a..041d94f7b 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py
@@ -1508,8 +1508,9 @@ def init_command(
console=console,
) as persist_bar:
persist_callback = callback.rebind(RichProgressCallback(persist_bar, console))
- # Announced before the work starts and re-announced with a real total
- # once the page loop knows one, so the stage is never silent.
+ # Indeterminate: persistence has no page-proportional loop to count
+ # since the full-text index became a single statement. Announced here
+ # so the stage is never silent.
persist_callback.on_phase_start("persist", None)
run_async(persist_result(result, repo_path, persist_callback, callback.table))
persist_callback.on_phase_done("persist")
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index 1a370790f..c41ddd9ea 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -94,14 +94,12 @@ async def _index_preserved_pages(sf: Any, fts: Any, preserved_page_ids: set[str]
).where(Page.id.in_(batch))
)
).all()
- for page_id, title, content, summary, target_path in rows:
- await fts.index(
- page_id,
- title or "",
- content or "",
- summary=summary or "",
- target_path=target_path or "",
- )
+ await fts.index_many(
+ [
+ (page_id, title or "", content or "", summary or "", target_path or "")
+ for page_id, title, content, summary, target_path in rows
+ ]
+ )
except Exception as exc: # pragma: no cover - defensive
logger.debug("persist.preserved_fts_backfill_failed", error=str(exc))
@@ -116,11 +114,11 @@ async def persist_result(
Handles both index-only (no pages) and full (with pages + FTS) modes.
- *progress* is optional and reports the full-text indexing loop below, the
- one part of persistence whose length is known in advance and proportional
- to the wiki. Everything after "Generated N pages" used to happen under a
- single indeterminate spinner, so on a repo of a few thousand pages the run
- sat silent for minutes with no way to tell work from a hang.
+ *progress* is optional, and closing the phase is all it is used for here.
+ The full-text index used to be written a page at a time, which was minutes
+ of silence on a large wiki and earned a bar with a real denominator; it is
+ one statement now, and nothing else persistence does is proportional to
+ the page count, so the phase the CLI opens stays indeterminate.
*timings* splits that span into its SQL and full-text halves, which the
single ``persist`` number cannot distinguish.
@@ -253,18 +251,12 @@ async def persist_result(
if fts is not None and swept_page_ids:
await fts.delete_many(swept_page_ids)
if fts is not None and result.generated_pages:
- if progress is not None:
- progress.on_phase_start("persist", len(result.generated_pages))
- for page in result.generated_pages:
- await fts.index(
- page.page_id,
- page.title,
- page.content,
- summary=page.summary,
- target_path=page.target_path,
- )
- if progress is not None:
- progress.on_item_done("persist")
+ await fts.index_many(
+ [
+ (page.page_id, page.title, page.content, page.summary, page.target_path)
+ for page in result.generated_pages
+ ]
+ )
await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))
# Stamp the analysis (+ generation) phases in the resume ledger now that
diff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.py
index b7af3f267..2ede5bdf7 100644
--- a/packages/core/src/repowise/core/persistence/search.py
+++ b/packages/core/src/repowise/core/persistence/search.py
@@ -18,8 +18,9 @@ from __future__ import annotations
import logging
from collections import OrderedDict
+from collections.abc import Sequence
from dataclasses import dataclass
-from typing import Literal
+from typing import Any, Literal
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.sql import text
@@ -67,6 +68,31 @@ PAGE_FTS_DDL = (
"USING fts5(page_id UNINDEXED, title, content, summary, target_path)"
)
+_PAGE_FTS_INSERT_SQL = (
+ "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
+ "VALUES (:pid, :title, :content, :summary, :target_path)"
+)
+
+# SQLite allows 999 host parameters per statement by default.
+_ID_CHUNK = 500
+
+
+async def _delete_page_ids(conn: Any, page_ids: Sequence[str]) -> None:
+ """Delete the rows for *page_ids* on an already-open connection.
+
+ Takes the connection rather than opening one so a caller that also
+ inserts can put both halves in the same transaction.
+ """
+ for start in range(0, len(page_ids), _ID_CHUNK):
+ chunk = page_ids[start : start + _ID_CHUNK]
+ placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))
+ params = {f"p{i}": pid for i, pid in enumerate(chunk)}
+ await conn.execute(
+ text(f"DELETE FROM page_fts WHERE page_id IN ({placeholders})"),
+ params,
+ )
+
+
# An indexed row whose page is gone from ``wiki_pages``. Counting and deleting
# share the predicate so the number reported is exactly the number removed.
# ``NOT EXISTS`` rather than ``NOT IN``: the subquery's column is a primary key
@@ -368,47 +394,68 @@ class FullTextSearch:
A page below the information floor is not indexed, and any row it
already had is deleted. Ten places write this index, so the decision
- lives here rather than at each of them — one that skipped the check
- would keep re-admitting the pages the others exclude, and nothing
- would report the disagreement. The page itself is untouched: it stays
- in ``wiki_pages`` and stays a valid link target.
+ lives in :meth:`index_many` rather than at each of them — one that
+ skipped the check would keep re-admitting the pages the others
+ exclude, and nothing would report the disagreement. The page itself is
+ untouched: it stays in ``wiki_pages`` and stays a valid link target.
"""
- if summary is None or target_path is None:
- self._warn_missing_index_fields(page_id, summary, target_path)
- summary = summary if summary is not None else ""
- target_path = target_path if target_path is not None else ""
+ await self.index_many([(page_id, title, content, summary, target_path)])
- indexable = meets_information_floor(content)
- if not indexable:
- self._count_skipped_below_floor(page_id, content)
+ async def index_many(
+ self,
+ pages: Sequence[tuple[str, str, str, str | None, str | None]],
+ ) -> None:
+ """Add or replace many pages in a single transaction.
- if self._dialect == "sqlite":
- async with self._engine.begin() as conn:
- # FTS5 does not support UPDATE; use DELETE + INSERT. The delete
- # runs either way: a page that has fallen below the floor since
- # its last index must lose the row it had, or the exclusion
- # only ever applies to pages that never existed.
- await conn.execute(
- text("DELETE FROM page_fts WHERE page_id = :pid"),
- {"pid": page_id},
- )
- if not indexable:
- return
- await conn.execute(
- text(
- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "
- "VALUES (:pid, :title, :content, :summary, :target_path)"
- ),
- {
- "pid": page_id,
- "title": title,
- "content": content,
- "summary": summary,
- "target_path": target_path,
- },
- )
- # PostgreSQL: the GIN index on wiki_pages is maintained automatically
- # by the database as rows are inserted/updated via the CRUD layer.
+ Each entry is ``(page_id, title, content, summary, target_path)`` and
+ gets exactly the semantics :meth:`index` documents, floor exclusion
+ included — this is the one implementation and :meth:`index` is the
+ single-page call into it. What changes is the transaction count:
+ indexing a wiki a page at a time costs one commit per page, which on a
+ few thousand pages is most of what persisting the index takes.
+
+ A page id repeated inside *pages* keeps its last entry, which is what
+ the per-page loop this replaces would have left on disk.
+ """
+ if not pages:
+ return
+
+ # The accounting runs on every dialect, because a call site that
+ # forgets a field or writes a page too thin to index is wrong wherever
+ # it runs, and only these counters say so.
+ #
+ # FTS5 has no UPDATE, so a write is DELETE + INSERT. Every id is
+ # deleted and only the qualifying ones are re-inserted: a page that
+ # has fallen below the floor since its last index has to lose the row
+ # it had, or the exclusion only ever applies to pages that never
+ # existed.
+ deletions: dict[str, None] = {}
+ insertions: dict[str, dict[str, str]] = {}
+ for page_id, title, content, summary, target_path in pages:
+ deletions[page_id] = None
+ if summary is None or target_path is None:
+ self._warn_missing_index_fields(page_id, summary, target_path)
+ if meets_information_floor(content):
+ insertions[page_id] = {
+ "pid": page_id,
+ "title": title,
+ "content": content,
+ "summary": summary or "",
+ "target_path": target_path or "",
+ }
+ else:
+ self._count_skipped_below_floor(page_id, content)
+ insertions.pop(page_id, None)
+
+ if self._dialect != "sqlite":
+ # PostgreSQL: the GIN index on wiki_pages is maintained
+ # automatically as rows are written through the CRUD layer.
+ return
+
+ async with self._engine.begin() as conn:
+ await _delete_page_ids(conn, list(deletions))
+ if insertions:
+ await conn.execute(text(_PAGE_FTS_INSERT_SQL), list(insertions.values()))
def _count_skipped_below_floor(self, page_id: str, content: str) -> None:
"""Record a page held out of the index, and name the first few."""
@@ -475,17 +522,8 @@ class FullTextSearch:
if not page_ids:
return
if self._dialect == "sqlite":
- # Chunk to stay under SQLite's default 999-parameter limit per statement.
- chunk_size = 500
async with self._engine.begin() as conn:
- for start in range(0, len(page_ids), chunk_size):
- chunk = page_ids[start : start + chunk_size]
- placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))
- params = {f"p{i}": pid for i, pid in enumerate(chunk)}
- await conn.execute(
- text(f"DELETE FROM page_fts WHERE page_id IN ({placeholders})"),
- params,
- )
+ await _delete_page_ids(conn, page_ids)
# PostgreSQL: rows deleted via CASCADE automatically remove from GIN index.
async def list_indexed_ids(self) -> set[str]:
diff --git a/tests/unit/cli/test_persist_result_progress.py b/tests/unit/cli/test_persist_result_progress.py
index 38751b85a..a3ed32aeb 100644
--- a/tests/unit/cli/test_persist_result_progress.py
+++ b/tests/unit/cli/test_persist_result_progress.py
@@ -5,6 +5,11 @@ SQL upserts, the stale-page sweep, and a full-text index loop that awaits once
per generated page. On a few thousand pages that is minutes of a screen that
looks identical whether the run is working or wedged — directly after a
generation bar that had just announced it was finished.
+
+The full-text index is one statement now, so the loop that earned a real
+denominator is gone and what remains is bounded by the SQL half. The phase is
+still announced and still closed; it is the per-page count that no longer
+describes anything.
"""
from __future__ import annotations
@@ -72,7 +77,12 @@ def _result(pages: list[GeneratedPage]) -> SimpleNamespace:
)
-async def test_full_text_indexing_reports_a_page_at_a_time(tmp_path):
+async def test_persistence_claims_no_per_page_progress_it_cannot_report(tmp_path):
+ """Counting pages here would fill the bar the instant the batch landed.
+
+ The rest of persistence would then run behind a bar reading 100%, which
+ misreads as a hang the same way the old silent spinner did.
+ """
repo_path = tmp_path / "repo"
repo_path.mkdir()
progress = _RecordingProgress()
@@ -80,9 +90,8 @@ async def test_full_text_indexing_reports_a_page_at_a_time(tmp_path):
pages = [_page("a"), _page("b"), _page("c")]
await persist_result(_result(pages), repo_path, progress)
- # A real denominator, not a spinner: the loop length is known up front.
- assert ("persist", len(pages)) in progress.started
- assert progress.items.count("persist") == len(pages)
+ assert [phase for phase, _ in progress.started if phase == "persist"] == []
+ assert progress.items.count("persist") == 0
assert "persist" in progress.done
diff --git a/tests/unit/persistence/test_information_floor.py b/tests/unit/persistence/test_information_floor.py
index 6e9fffa45..cfcb9f7e8 100644
--- a/tests/unit/persistence/test_information_floor.py
+++ b/tests/unit/persistence/test_information_floor.py
@@ -285,3 +285,37 @@ def test_the_floor_is_off_unless_it_is_set(monkeypatch):
assert information_floor() == 0
assert meets_information_floor(SKELETON) is True
assert meets_information_floor("") is True
+
+
+async def test_a_batch_holds_out_only_the_thin_pages(fts):
+ await fts.index_many(
+ [
+ ("file_page:esbuild.mjs", "File: esbuild.mjs", SKELETON, "", "esbuild.mjs"),
+ ("file_page:search.py", "File: search.py", SUBSTANTIAL, "", "search.py"),
+ ]
+ )
+
+ assert await fts.list_indexed_ids() == {"file_page:search.py"}
+ assert fts.skipped_below_floor == 1
+
+
+async def test_a_batch_drops_the_row_of_a_page_that_thinned_out(fts):
+ """The delete runs for every id in the batch, insert only for those above."""
+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")
+
+ await fts.index_many([("file_page:a.py", "File: a.py", SKELETON, "", "a.py")])
+
+ assert await fts.list_indexed_ids() == set()
+ assert await fts.search("FTS5 virtual table") == []
+
+
+async def test_a_batch_whose_last_entry_is_thin_leaves_no_row(fts):
+ """Last-entry-wins has to hold when the last entry is the excluded one."""
+ await fts.index_many(
+ [
+ ("file_page:a.py", "File: a.py", SUBSTANTIAL, "", "a.py"),
+ ("file_page:a.py", "File: a.py", SKELETON, "", "a.py"),
+ ]
+ )
+
+ assert await fts.list_indexed_ids() == set()
diff --git a/tests/unit/persistence/test_search.py b/tests/unit/persistence/test_search.py
index d1d64768a..9eb32be5d 100644
--- a/tests/unit/persistence/test_search.py
+++ b/tests/unit/persistence/test_search.py
@@ -143,3 +143,70 @@ async def test_full_text_search_multiple_pages_relevance_ordered(fts):
# Both should appear; 'exact' should rank first (title match)
assert len(results) >= 1
assert results[0].page_id == "exact"
+
+
+# ---------------------------------------------------------------------------
+# Batch indexing
+# ---------------------------------------------------------------------------
+
+
+async def _indexed_ids(fts) -> set[str]:
+ return await fts.list_indexed_ids()
+
+
+async def test_index_many_writes_the_same_rows_as_indexing_one_at_a_time(async_engine):
+ """The batch path is the single-page path, so the two must not diverge."""
+ pages = [
+ (f"p{i}", f"Title {i}", f"content about widgets number {i}", f"sum {i}", f"a/b{i}.py")
+ for i in range(20)
+ ]
+
+ one_at_a_time = FullTextSearch(async_engine)
+ await one_at_a_time.ensure_index()
+ for page in pages:
+ await one_at_a_time.index(page[0], page[1], page[2], summary=page[3], target_path=page[4])
+ sequential = sorted(
+ (r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50)
+ )
+
+ await one_at_a_time.delete_many([p[0] for p in pages])
+ await one_at_a_time.index_many(pages)
+ batched = sorted((r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50))
+
+ assert batched == sequential
+ assert len(batched) == 20
+
+
+async def test_index_many_spans_more_ids_than_one_statement_can_bind(fts):
+ """The delete half chunks; 1,200 ids is past both chunk boundaries.
+
+ A wiki is thousands of pages, so the first real corpus would have been the
+ first test of this path.
+ """
+ pages = [(f"p{i}", f"Title {i}", "a page about chunking", "", f"{i}.py") for i in range(1200)]
+
+ await fts.index_many(pages)
+ assert len(await _indexed_ids(fts)) == 1200
+
+ # Re-indexing the same ids must replace rather than duplicate them.
+ await fts.index_many(pages)
+ assert len(await _indexed_ids(fts)) == 1200
+
+
+async def test_a_page_id_repeated_in_one_batch_keeps_its_last_entry(fts):
+ """Every id is deleted before any is inserted, so a duplicate would double."""
+ await fts.index_many(
+ [
+ ("p1", "First", "a page about alpacas", "", "a.py"),
+ ("p1", "Second", "a page about zebras", "", "a.py"),
+ ]
+ )
+
+ assert await _indexed_ids(fts) == {"p1"}
+ assert [r.page_id for r in await fts.search("zebras")] == ["p1"]
+ assert await fts.search("alpacas") == []
+
+
+async def test_index_many_of_nothing_is_a_no_op(fts):
+ await fts.index_many([])
+ assert await _indexed_ids(fts) == set()