Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(persist): write the full-text index in one transaction (#2052)
Raghav Chamadiya07abb3cba727
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/command.py b/packages/cli/src/repowise/cli/commands/init_cmd/command.pyindex 0fe07132a..041d94f7b 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/command.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/command.py@@ -1508,8 +1508,9 @@ def init_command(console=console,) as persist_bar:persist_callback = callback.rebind(RichProgressCallback(persist_bar, console))- # Announced before the work starts and re-announced with a real total- # once the page loop knows one, so the stage is never silent.+ # Indeterminate: persistence has no page-proportional loop to count+ # since the full-text index became a single statement. Announced here+ # so the stage is never silent.persist_callback.on_phase_start("persist", None)run_async(persist_result(result, repo_path, persist_callback, callback.table))persist_callback.on_phase_done("persist")diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex 1a370790f..c41ddd9ea 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -94,14 +94,12 @@ async def _index_preserved_pages(sf: Any, fts: Any, preserved_page_ids: set[str]).where(Page.id.in_(batch)))).all()- for page_id, title, content, summary, target_path in rows:- await fts.index(- page_id,- title or "",- content or "",- summary=summary or "",- target_path=target_path or "",- )+ await fts.index_many(+ [+ (page_id, title or "", content or "", summary or "", target_path or "")+ for page_id, title, content, summary, target_path in rows+ ]+ )except Exception as exc: # pragma: no cover - defensivelogger.debug("persist.preserved_fts_backfill_failed", error=str(exc))@@ -116,11 +114,11 @@ async def persist_result(Handles both index-only (no pages) and full (with pages + FTS) modes.- *progress* is optional and reports the full-text indexing loop below, the- one part of persistence whose length is known in advance and proportional- to the wiki. Everything after "Generated N pages" used to happen under a- single indeterminate spinner, so on a repo of a few thousand pages the run- sat silent for minutes with no way to tell work from a hang.+ *progress* is optional, and closing the phase is all it is used for here.+ The full-text index used to be written a page at a time, which was minutes+ of silence on a large wiki and earned a bar with a real denominator; it is+ one statement now, and nothing else persistence does is proportional to+ the page count, so the phase the CLI opens stays indeterminate.*timings* splits that span into its SQL and full-text halves, which thesingle ``persist`` number cannot distinguish.@@ -253,18 +251,12 @@ async def persist_result(if fts is not None and swept_page_ids:await fts.delete_many(swept_page_ids)if fts is not None and result.generated_pages:- if progress is not None:- progress.on_phase_start("persist", len(result.generated_pages))- for page in result.generated_pages:- await fts.index(- page.page_id,- page.title,- page.content,- summary=page.summary,- target_path=page.target_path,- )- if progress is not None:- progress.on_item_done("persist")+ await fts.index_many(+ [+ (page.page_id, page.title, page.content, page.summary, page.target_path)+ for page in result.generated_pages+ ]+ )await _index_preserved_pages(sf, fts, getattr(result, "preserved_page_ids", None))# Stamp the analysis (+ generation) phases in the resume ledger now thatdiff --git a/packages/core/src/repowise/core/persistence/search.py b/packages/core/src/repowise/core/persistence/search.pyindex b7af3f267..2ede5bdf7 100644--- a/packages/core/src/repowise/core/persistence/search.py+++ b/packages/core/src/repowise/core/persistence/search.py@@ -18,8 +18,9 @@ from __future__ import annotationsimport loggingfrom collections import OrderedDict+from collections.abc import Sequencefrom dataclasses import dataclass-from typing import Literal+from typing import Any, Literalfrom sqlalchemy.ext.asyncio import AsyncEnginefrom sqlalchemy.sql import text@@ -67,6 +68,31 @@ PAGE_FTS_DDL = ("USING fts5(page_id UNINDEXED, title, content, summary, target_path)")+_PAGE_FTS_INSERT_SQL = (+ "INSERT INTO page_fts(page_id, title, content, summary, target_path) "+ "VALUES (:pid, :title, :content, :summary, :target_path)"+)++# SQLite allows 999 host parameters per statement by default.+_ID_CHUNK = 500+++async def _delete_page_ids(conn: Any, page_ids: Sequence[str]) -> None:+ """Delete the rows for *page_ids* on an already-open connection.++ Takes the connection rather than opening one so a caller that also+ inserts can put both halves in the same transaction.+ """+ for start in range(0, len(page_ids), _ID_CHUNK):+ chunk = page_ids[start : start + _ID_CHUNK]+ placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))+ params = {f"p{i}": pid for i, pid in enumerate(chunk)}+ await conn.execute(+ text(f"DELETE FROM page_fts WHERE page_id IN ({placeholders})"),+ params,+ )++# An indexed row whose page is gone from ``wiki_pages``. Counting and deleting# share the predicate so the number reported is exactly the number removed.# ``NOT EXISTS`` rather than ``NOT IN``: the subquery's column is a primary key@@ -368,47 +394,68 @@ class FullTextSearch:A page below the information floor is not indexed, and any row italready had is deleted. Ten places write this index, so the decision- lives here rather than at each of them — one that skipped the check- would keep re-admitting the pages the others exclude, and nothing- would report the disagreement. The page itself is untouched: it stays- in ``wiki_pages`` and stays a valid link target.+ lives in :meth:`index_many` rather than at each of them — one that+ skipped the check would keep re-admitting the pages the others+ exclude, and nothing would report the disagreement. The page itself is+ untouched: it stays in ``wiki_pages`` and stays a valid link target."""- if summary is None or target_path is None:- self._warn_missing_index_fields(page_id, summary, target_path)- summary = summary if summary is not None else ""- target_path = target_path if target_path is not None else ""+ await self.index_many([(page_id, title, content, summary, target_path)])- indexable = meets_information_floor(content)- if not indexable:- self._count_skipped_below_floor(page_id, content)+ async def index_many(+ self,+ pages: Sequence[tuple[str, str, str, str | None, str | None]],+ ) -> None:+ """Add or replace many pages in a single transaction.- if self._dialect == "sqlite":- async with self._engine.begin() as conn:- # FTS5 does not support UPDATE; use DELETE + INSERT. The delete- # runs either way: a page that has fallen below the floor since- # its last index must lose the row it had, or the exclusion- # only ever applies to pages that never existed.- await conn.execute(- text("DELETE FROM page_fts WHERE page_id = :pid"),- {"pid": page_id},- )- if not indexable:- return- await conn.execute(- text(- "INSERT INTO page_fts(page_id, title, content, summary, target_path) "- "VALUES (:pid, :title, :content, :summary, :target_path)"- ),- {- "pid": page_id,- "title": title,- "content": content,- "summary": summary,- "target_path": target_path,- },- )- # PostgreSQL: the GIN index on wiki_pages is maintained automatically- # by the database as rows are inserted/updated via the CRUD layer.+ Each entry is ``(page_id, title, content, summary, target_path)`` and+ gets exactly the semantics :meth:`index` documents, floor exclusion+ included — this is the one implementation and :meth:`index` is the+ single-page call into it. What changes is the transaction count:+ indexing a wiki a page at a time costs one commit per page, which on a+ few thousand pages is most of what persisting the index takes.++ A page id repeated inside *pages* keeps its last entry, which is what+ the per-page loop this replaces would have left on disk.+ """+ if not pages:+ return++ # The accounting runs on every dialect, because a call site that+ # forgets a field or writes a page too thin to index is wrong wherever+ # it runs, and only these counters say so.+ #+ # FTS5 has no UPDATE, so a write is DELETE + INSERT. Every id is+ # deleted and only the qualifying ones are re-inserted: a page that+ # has fallen below the floor since its last index has to lose the row+ # it had, or the exclusion only ever applies to pages that never+ # existed.+ deletions: dict[str, None] = {}+ insertions: dict[str, dict[str, str]] = {}+ for page_id, title, content, summary, target_path in pages:+ deletions[page_id] = None+ if summary is None or target_path is None:+ self._warn_missing_index_fields(page_id, summary, target_path)+ if meets_information_floor(content):+ insertions[page_id] = {+ "pid": page_id,+ "title": title,+ "content": content,+ "summary": summary or "",+ "target_path": target_path or "",+ }+ else:+ self._count_skipped_below_floor(page_id, content)+ insertions.pop(page_id, None)++ if self._dialect != "sqlite":+ # PostgreSQL: the GIN index on wiki_pages is maintained+ # automatically as rows are written through the CRUD layer.+ return++ async with self._engine.begin() as conn:+ await _delete_page_ids(conn, list(deletions))+ if insertions:+ await conn.execute(text(_PAGE_FTS_INSERT_SQL), list(insertions.values()))def _count_skipped_below_floor(self, page_id: str, content: str) -> None:"""Record a page held out of the index, and name the first few."""@@ -475,17 +522,8 @@ class FullTextSearch:if not page_ids:returnif self._dialect == "sqlite":- # Chunk to stay under SQLite's default 999-parameter limit per statement.- chunk_size = 500async with self._engine.begin() as conn:- for start in range(0, len(page_ids), chunk_size):- chunk = page_ids[start : start + chunk_size]- placeholders = ", ".join(f":p{i}" for i in range(len(chunk)))- params = {f"p{i}": pid for i, pid in enumerate(chunk)}- await conn.execute(- text(f"DELETE FROM page_fts WHERE page_id IN ({placeholders})"),- params,- )+ await _delete_page_ids(conn, page_ids)# PostgreSQL: rows deleted via CASCADE automatically remove from GIN index.async def list_indexed_ids(self) -> set[str]:diff --git a/tests/unit/cli/test_persist_result_progress.py b/tests/unit/cli/test_persist_result_progress.pyindex 38751b85a..a3ed32aeb 100644--- a/tests/unit/cli/test_persist_result_progress.py+++ b/tests/unit/cli/test_persist_result_progress.py@@ -5,6 +5,11 @@ SQL upserts, the stale-page sweep, and a full-text index loop that awaits onceper generated page. On a few thousand pages that is minutes of a screen thatlooks identical whether the run is working or wedged — directly after ageneration bar that had just announced it was finished.++The full-text index is one statement now, so the loop that earned a real+denominator is gone and what remains is bounded by the SQL half. The phase is+still announced and still closed; it is the per-page count that no longer+describes anything."""from __future__ import annotations@@ -72,7 +77,12 @@ def _result(pages: list[GeneratedPage]) -> SimpleNamespace:)-async def test_full_text_indexing_reports_a_page_at_a_time(tmp_path):+async def test_persistence_claims_no_per_page_progress_it_cannot_report(tmp_path):+ """Counting pages here would fill the bar the instant the batch landed.++ The rest of persistence would then run behind a bar reading 100%, which+ misreads as a hang the same way the old silent spinner did.+ """repo_path = tmp_path / "repo"repo_path.mkdir()progress = _RecordingProgress()@@ -80,9 +90,8 @@ async def test_full_text_indexing_reports_a_page_at_a_time(tmp_path):pages = [_page("a"), _page("b"), _page("c")]await persist_result(_result(pages), repo_path, progress)- # A real denominator, not a spinner: the loop length is known up front.- assert ("persist", len(pages)) in progress.started- assert progress.items.count("persist") == len(pages)+ assert [phase for phase, _ in progress.started if phase == "persist"] == []+ assert progress.items.count("persist") == 0assert "persist" in progress.donediff --git a/tests/unit/persistence/test_information_floor.py b/tests/unit/persistence/test_information_floor.pyindex 6e9fffa45..cfcb9f7e8 100644--- a/tests/unit/persistence/test_information_floor.py+++ b/tests/unit/persistence/test_information_floor.py@@ -285,3 +285,37 @@ def test_the_floor_is_off_unless_it_is_set(monkeypatch):assert information_floor() == 0assert meets_information_floor(SKELETON) is Trueassert meets_information_floor("") is True+++async def test_a_batch_holds_out_only_the_thin_pages(fts):+ await fts.index_many(+ [+ ("file_page:esbuild.mjs", "File: esbuild.mjs", SKELETON, "", "esbuild.mjs"),+ ("file_page:search.py", "File: search.py", SUBSTANTIAL, "", "search.py"),+ ]+ )++ assert await fts.list_indexed_ids() == {"file_page:search.py"}+ assert fts.skipped_below_floor == 1+++async def test_a_batch_drops_the_row_of_a_page_that_thinned_out(fts):+ """The delete runs for every id in the batch, insert only for those above."""+ await fts.index("file_page:a.py", "File: a.py", SUBSTANTIAL, summary="", target_path="a.py")++ await fts.index_many([("file_page:a.py", "File: a.py", SKELETON, "", "a.py")])++ assert await fts.list_indexed_ids() == set()+ assert await fts.search("FTS5 virtual table") == []+++async def test_a_batch_whose_last_entry_is_thin_leaves_no_row(fts):+ """Last-entry-wins has to hold when the last entry is the excluded one."""+ await fts.index_many(+ [+ ("file_page:a.py", "File: a.py", SUBSTANTIAL, "", "a.py"),+ ("file_page:a.py", "File: a.py", SKELETON, "", "a.py"),+ ]+ )++ assert await fts.list_indexed_ids() == set()diff --git a/tests/unit/persistence/test_search.py b/tests/unit/persistence/test_search.pyindex d1d64768a..9eb32be5d 100644--- a/tests/unit/persistence/test_search.py+++ b/tests/unit/persistence/test_search.py@@ -143,3 +143,70 @@ async def test_full_text_search_multiple_pages_relevance_ordered(fts):# Both should appear; 'exact' should rank first (title match)assert len(results) >= 1assert results[0].page_id == "exact"+++# ---------------------------------------------------------------------------+# Batch indexing+# ---------------------------------------------------------------------------+++async def _indexed_ids(fts) -> set[str]:+ return await fts.list_indexed_ids()+++async def test_index_many_writes_the_same_rows_as_indexing_one_at_a_time(async_engine):+ """The batch path is the single-page path, so the two must not diverge."""+ pages = [+ (f"p{i}", f"Title {i}", f"content about widgets number {i}", f"sum {i}", f"a/b{i}.py")+ for i in range(20)+ ]++ one_at_a_time = FullTextSearch(async_engine)+ await one_at_a_time.ensure_index()+ for page in pages:+ await one_at_a_time.index(page[0], page[1], page[2], summary=page[3], target_path=page[4])+ sequential = sorted(+ (r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50)+ )++ await one_at_a_time.delete_many([p[0] for p in pages])+ await one_at_a_time.index_many(pages)+ batched = sorted((r.page_id, r.title) for r in await one_at_a_time.search("widgets", limit=50))++ assert batched == sequential+ assert len(batched) == 20+++async def test_index_many_spans_more_ids_than_one_statement_can_bind(fts):+ """The delete half chunks; 1,200 ids is past both chunk boundaries.++ A wiki is thousands of pages, so the first real corpus would have been the+ first test of this path.+ """+ pages = [(f"p{i}", f"Title {i}", "a page about chunking", "", f"{i}.py") for i in range(1200)]++ await fts.index_many(pages)+ assert len(await _indexed_ids(fts)) == 1200++ # Re-indexing the same ids must replace rather than duplicate them.+ await fts.index_many(pages)+ assert len(await _indexed_ids(fts)) == 1200+++async def test_a_page_id_repeated_in_one_batch_keeps_its_last_entry(fts):+ """Every id is deleted before any is inserted, so a duplicate would double."""+ await fts.index_many(+ [+ ("p1", "First", "a page about alpacas", "", "a.py"),+ ("p1", "Second", "a page about zebras", "", "a.py"),+ ]+ )++ assert await _indexed_ids(fts) == {"p1"}+ assert [r.page_id for r in await fts.search("zebras")] == ["p1"]+ assert await fts.search("alpacas") == []+++async def test_index_many_of_nothing_is_a_no_op(fts):+ await fts.index_many([])+ assert await _indexed_ids(fts) == set()