Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(persistence): analyze the SQLite index at the end of every write (#4229)
Raghav Chamadiya241bb563778f
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.pyindex 2c7722370..558943aa3 100644--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py@@ -141,7 +141,12 @@ async def persist_result("""from datetime import UTC, datetime- from repowise.core.persistence import FullTextSearch, get_session, upsert_repository+ from repowise.core.persistence import (+ FullTextSearch,+ get_session,+ refresh_planner_stats,+ upsert_repository,+ )from repowise.core.persistence.crud import upsert_generation_jobfrom repowise.core.pipeline import (persist_analysis,@@ -297,6 +302,8 @@ async def persist_result(if result.generated_pages:await ledger.mark_completed(ResumePhase.GENERATION)+ with timed(timings, "persist.planner_stats"):+ await refresh_planner_stats(engine)await engine.dispose()# Closed here rather than after the indexing loop: the preserved-page# backfill and the ledger stamp are persistence too, and a phase thatdiff --git a/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.pyindex 03b7c0e67..a9c44bad7 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py@@ -448,6 +448,7 @@ async def write_full_update(ctx: FullUpdatePersist) -> int:create_session_factory,get_session,init_db,+ refresh_planner_stats,upsert_repository,)from repowise.core.pipeline.cleanup_debt import load_cleanup_debt@@ -487,6 +488,8 @@ async def write_full_update(ctx: FullUpdatePersist) -> int:with timed(timings, "persist.commit_health"):await refresh_commit_health_after_update(sf, ctx.repo_id, str(ctx.repo_path))+ with timed(timings, "persist.planner_stats"):+ await refresh_planner_stats(ctx.engine)return ctx.total_pagesfinally:await ctx.engine.dispose()diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.pyindex ebdcb2724..837516184 100644--- a/packages/core/src/repowise/core/persistence/__init__.py+++ b/packages/core/src/repowise/core/persistence/__init__.py@@ -111,6 +111,7 @@ from .database import (get_repo_db_path,get_session,init_db,+ refresh_planner_stats,resolve_db_url,)from .decision_graph import (@@ -284,6 +285,7 @@ __all__ = ["load_stale_update_targets","mark_webhook_processed","recompute_decision_staleness",+ "refresh_planner_stats","resolve_db_url","save_dead_code_findings","store_webhook_event",diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.pyindex 5d7af1952..c8cb5701c 100644--- a/packages/core/src/repowise/core/persistence/database.py+++ b/packages/core/src/repowise/core/persistence/database.py@@ -21,6 +21,7 @@ from typing import Anyimport structlogfrom sqlalchemy import String, event, inspect, literal+from sqlalchemy.exc import DBAPIErrorfrom sqlalchemy.ext.asyncio import (AsyncEngine,AsyncSession,@@ -167,6 +168,7 @@ __all__ = ["get_repo_db_path","get_session","init_db",+ "refresh_planner_stats","resolve_db_url","schema_behind",]@@ -706,6 +708,67 @@ async def init_db(engine: AsyncEngine) -> None:await conn.execute(text(PAGE_FTS_DDL))+# A table whose row count moved past this factor of the count ANALYZE recorded+# has statistics too old to plan by.+_STATS_DRIFT = 2+++def _planner_stats_stale(conn: Any) -> bool:+ """Whether a table with rows has no statistics, or its count left the band."""+ # ANALYZE skips virtual tables (the full-text index), so they never count.+ names = [+ r[0]+ for r in conn.exec_driver_sql(+ "SELECT name FROM sqlite_master WHERE type = 'table' "+ "AND sql NOT LIKE 'CREATE VIRTUAL TABLE%'"+ )+ ]+ if "sqlite_stat1" not in names:+ return True+ recorded = dict(+ conn.exec_driver_sql(+ "SELECT tbl, MAX(CAST(stat AS INTEGER)) FROM sqlite_stat1 GROUP BY tbl"+ ).all()+ )+ for name in names:+ if name.startswith("sqlite_"):+ continue+ # One statement per table on purpose: SQLite answers a bare COUNT(*)+ # from the b-tree, but scans the rows when the count sits inside a+ # subquery, which made one UNION ALL about 25x slower.+ rows = conn.exec_driver_sql(f'SELECT COUNT(*) FROM "{name}"').scalar() or 0+ before = recorded.get(name)+ if before is None:+ if rows:+ return True+ elif not before / _STATS_DRIFT <= rows <= before * _STATS_DRIFT:+ return True+ return False+++async def refresh_planner_stats(engine: AsyncEngine) -> None:+ """Refresh SQLite's planner statistics at the end of a write to the index.++ Without ``sqlite_stat1`` the planner guesses, and on a large store it picks+ a wide covering index over the selective one (a full edge-table scan in+ place of an index lookup). A full ``ANALYZE``, not a sampled one: every+ index leads with ``repository_id``, which a sample (``analysis_limit``, and+ ``PRAGMA optimize`` since SQLite 3.46) reads as selective, so the planner+ keeps the wrong index. It runs only when the statistics are missing or a+ table's size has drifted, since a full pass reads the whole store. Other+ backends keep their own statistics. A failure costs only a slower plan, so+ it never fails the write.+ """+ if engine.dialect.name != "sqlite":+ return+ try:+ async with engine.begin() as conn:+ if await conn.run_sync(_planner_stats_stale):+ await conn.execute(text("ANALYZE"))+ except DBAPIError as exc: # a locked or read-only store+ log.warning("sqlite.analyze_failed", error=str(exc))++async def schema_behind(engine: AsyncEngine) -> bool:"""Whether a table or column the models declare is missing from the store.diff --git a/packages/core/src/repowise/core/pipeline/full_index.py b/packages/core/src/repowise/core/pipeline/full_index.pyindex d0091df84..55d139048 100644--- a/packages/core/src/repowise/core/pipeline/full_index.py+++ b/packages/core/src/repowise/core/pipeline/full_index.py@@ -38,6 +38,7 @@ async def index_repo_full(create_session_factory,get_session,init_db,+ refresh_planner_stats,upsert_repository,)from repowise.core.persistence.database import resolve_db_url@@ -90,6 +91,7 @@ async def index_repo_full(from repowise.core.persistence.crud.analysis.actions import write_read_snapshotsawait write_read_snapshots(session, repo.id)+ await refresh_planner_stats(engine)from repowise.core.pipeline.cleanup_debt import (clear_cleanup_debt,diff --git a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py b/packages/core/src/repowise/core/pipeline/incremental/persist_index.pyindex 951b9813a..3d47bd0bb 100644--- a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py+++ b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py@@ -603,6 +603,7 @@ async def _write(ctx: IncrementalPersist) -> None:create_session_factory,get_session,init_db,+ refresh_planner_stats,upsert_repository,)from repowise.core.persistence.database import resolve_db_url@@ -649,6 +650,8 @@ async def _write(ctx: IncrementalPersist) -> None:set(ctx.tombstoned_page_ids) | set(ctx.deselected_page_ids) | cleanup_debt["vectors"])await run_steps(ctx, CLEANUP_STEPS)+ with timed(timings, "persist.planner_stats"):+ await refresh_planner_stats(ctx.engine)# Ceiling: retired non-file pages' *vector* embeddings survive this path.# There is no store here to delete them from, and building one woulddiff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.pyindex 7cabd8e0f..0d267fe7e 100644--- a/packages/server/src/repowise/server/job_executor.py+++ b/packages/server/src/repowise/server/job_executor.py@@ -29,7 +29,7 @@ from repowise.core.persistence.crud import (get_repository,update_job_status,)-from repowise.core.persistence.database import get_session+from repowise.core.persistence.database import get_session, refresh_planner_statsfrom repowise.core.pipeline import persist_pipeline_result, run_pipelinefrom repowise.core.pipeline.modes import OrchestratorModefrom repowise.server.job_events import JobEventBuffer, create_event_buffer@@ -565,6 +565,7 @@ async def _run_index_job(run: _JobRun, app_state: Any) -> None:all_pages = (result.generated_pages or []) + incremental_pagesawait _refresh_fts(run.fts, swept_page_ids, all_pages)+ await refresh_planner_stats(run.session_factory.kw["bind"])# Stop progress updates first, so none overwrites "completed".await run.progress.drain_and_stop()diff --git a/tests/unit/persistence/test_database_pragmas.py b/tests/unit/persistence/test_database_pragmas.pyindex 9e6bc9b13..4884410ee 100644--- a/tests/unit/persistence/test_database_pragmas.py+++ b/tests/unit/persistence/test_database_pragmas.py@@ -285,3 +285,103 @@ def test_a_non_operational_pragma_failure_still_raises() -> None:with pytest.raises(sqlite3.ProgrammingError):listener(_Conn(), None)+++async def test_planner_stats_count_every_row_of_a_single_repository(tmp_path: Path) -> None:+ """Every index leads with ``repository_id``. A sampled ANALYZE reads that+ column as selective and the planner then scans the wide covering index for+ a target lookup, so the stats must come from the whole table."""+ from repowise.core.persistence import create_session_factory, refresh_planner_stats+ from repowise.core.persistence.models import GraphEdge, Repository++ db = tmp_path / "wiki.db"+ engine = create_engine(f"sqlite+aiosqlite:///{db}")+ try:+ await init_db(engine)+ async with create_session_factory(engine)() as session:+ session.add(Repository(id="r1", name="r", local_path="/r"))+ await session.flush()+ session.add_all(+ GraphEdge(repository_id="r1", source_node_id=f"s{i % 3}", target_node_id=f"t{i}")+ for i in range(3000)+ )+ await session.commit()+ await refresh_planner_stats(engine)+ finally:+ await engine.dispose()++ conn = sqlite3.connect(db)+ stat = conn.execute(+ "SELECT stat FROM sqlite_stat1 WHERE idx = 'ix_graph_edges_repo_target'"+ ).fetchone()+ plan = conn.execute(+ "EXPLAIN QUERY PLAN SELECT DISTINCT source_node_id, target_node_id FROM graph_edges "+ "WHERE repository_id = 'r1' AND target_node_id IN ('t1', 't2') AND edge_type IN ('imports')"+ ).fetchall()+ conn.close()+ assert stat is not None and stat[0].split()[:2] == ["3000", "3000"]+ assert "ix_graph_edges_repo_target" in str(plan)+++async def test_planner_stats_rerun_only_when_a_table_drifts(tmp_path: Path) -> None:+ """A full pass reads the whole store, so a write that leaves every table+ near the size ANALYZE recorded keeps the statistics it has."""+ from repowise.core.persistence import create_session_factory, refresh_planner_stats+ from repowise.core.persistence.models import GraphEdge, Repository++ db = tmp_path / "wiki.db"+ engine = create_engine(f"sqlite+aiosqlite:///{db}")++ async def add_edges(start: int, count: int) -> None:+ async with create_session_factory(engine)() as session:+ session.add_all(+ GraphEdge(repository_id="r1", source_node_id="s", target_node_id=f"t{i}")+ for i in range(start, start + count)+ )+ await session.commit()++ def recorded() -> str:+ conn = sqlite3.connect(db)+ try:+ return conn.execute(+ "SELECT stat FROM sqlite_stat1 WHERE idx = 'ix_graph_edges_repo_target'"+ ).fetchone()[0]+ finally:+ conn.close()++ try:+ await init_db(engine)+ async with create_session_factory(engine)() as session:+ session.add(Repository(id="r1", name="r", local_path="/r"))+ await session.commit()+ await add_edges(0, 100)+ await refresh_planner_stats(engine)+ await add_edges(100, 50)+ await refresh_planner_stats(engine)+ within_band = recorded()+ await add_edges(150, 100)+ await refresh_planner_stats(engine)+ drifted = recorded()+ finally:+ await engine.dispose()++ assert within_band.split()[0] == "100"+ assert drifted.split()[0] == "250"+++async def test_planner_stats_skip_other_backends() -> None:+ from repowise.core.persistence import refresh_planner_stats++ class _Dialect:+ name = "postgresql"++ class _Engine:+ dialect = _Dialect()++ def begin(self): # pragma: no cover - reaching it is the failure+ raise AssertionError("ANALYZE must not run off SQLite")++ await refresh_planner_stats(_Engine()) # type: ignore[arg-type]