Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(persistence): analyze the SQLite index at the end of every write (#4229)

Raghav Chamadiya241bb563778f
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
index 2c7722370..558943aa3 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd/persistence.py
@@ -141,7 +141,12 @@ async def persist_result(
"""
from datetime import UTC, datetime
- from repowise.core.persistence import FullTextSearch, get_session, upsert_repository
+ from repowise.core.persistence import (
+ FullTextSearch,
+ get_session,
+ refresh_planner_stats,
+ upsert_repository,
+ )
from repowise.core.persistence.crud import upsert_generation_job
from repowise.core.pipeline import (
persist_analysis,
@@ -297,6 +302,8 @@ async def persist_result(
if result.generated_pages:
await ledger.mark_completed(ResumePhase.GENERATION)
+ with timed(timings, "persist.planner_stats"):
+ await refresh_planner_stats(engine)
await engine.dispose()
# Closed here rather than after the indexing loop: the preserved-page
# backfill and the ledger stamp are persistence too, and a phase that
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py
index 03b7c0e67..a9c44bad7 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/full_persist.py
@@ -448,6 +448,7 @@ async def write_full_update(ctx: FullUpdatePersist) -> int:
create_session_factory,
get_session,
init_db,
+ refresh_planner_stats,
upsert_repository,
)
from repowise.core.pipeline.cleanup_debt import load_cleanup_debt
@@ -487,6 +488,8 @@ async def write_full_update(ctx: FullUpdatePersist) -> int:
with timed(timings, "persist.commit_health"):
await refresh_commit_health_after_update(sf, ctx.repo_id, str(ctx.repo_path))
+ with timed(timings, "persist.planner_stats"):
+ await refresh_planner_stats(ctx.engine)
return ctx.total_pages
finally:
await ctx.engine.dispose()
diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.py
index ebdcb2724..837516184 100644
--- a/packages/core/src/repowise/core/persistence/__init__.py
+++ b/packages/core/src/repowise/core/persistence/__init__.py
@@ -111,6 +111,7 @@ from .database import (
get_repo_db_path,
get_session,
init_db,
+ refresh_planner_stats,
resolve_db_url,
)
from .decision_graph import (
@@ -284,6 +285,7 @@ __all__ = [
"load_stale_update_targets",
"mark_webhook_processed",
"recompute_decision_staleness",
+ "refresh_planner_stats",
"resolve_db_url",
"save_dead_code_findings",
"store_webhook_event",
diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.py
index 5d7af1952..c8cb5701c 100644
--- a/packages/core/src/repowise/core/persistence/database.py
+++ b/packages/core/src/repowise/core/persistence/database.py
@@ -21,6 +21,7 @@ from typing import Any
import structlog
from sqlalchemy import String, event, inspect, literal
+from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
@@ -167,6 +168,7 @@ __all__ = [
"get_repo_db_path",
"get_session",
"init_db",
+ "refresh_planner_stats",
"resolve_db_url",
"schema_behind",
]
@@ -706,6 +708,67 @@ async def init_db(engine: AsyncEngine) -> None:
await conn.execute(text(PAGE_FTS_DDL))
+# A table whose row count moved past this factor of the count ANALYZE recorded
+# has statistics too old to plan by.
+_STATS_DRIFT = 2
+
+
+def _planner_stats_stale(conn: Any) -> bool:
+ """Whether a table with rows has no statistics, or its count left the band."""
+ # ANALYZE skips virtual tables (the full-text index), so they never count.
+ names = [
+ r[0]
+ for r in conn.exec_driver_sql(
+ "SELECT name FROM sqlite_master WHERE type = 'table' "
+ "AND sql NOT LIKE 'CREATE VIRTUAL TABLE%'"
+ )
+ ]
+ if "sqlite_stat1" not in names:
+ return True
+ recorded = dict(
+ conn.exec_driver_sql(
+ "SELECT tbl, MAX(CAST(stat AS INTEGER)) FROM sqlite_stat1 GROUP BY tbl"
+ ).all()
+ )
+ for name in names:
+ if name.startswith("sqlite_"):
+ continue
+ # One statement per table on purpose: SQLite answers a bare COUNT(*)
+ # from the b-tree, but scans the rows when the count sits inside a
+ # subquery, which made one UNION ALL about 25x slower.
+ rows = conn.exec_driver_sql(f'SELECT COUNT(*) FROM "{name}"').scalar() or 0
+ before = recorded.get(name)
+ if before is None:
+ if rows:
+ return True
+ elif not before / _STATS_DRIFT <= rows <= before * _STATS_DRIFT:
+ return True
+ return False
+
+
+async def refresh_planner_stats(engine: AsyncEngine) -> None:
+ """Refresh SQLite's planner statistics at the end of a write to the index.
+
+ Without ``sqlite_stat1`` the planner guesses, and on a large store it picks
+ a wide covering index over the selective one (a full edge-table scan in
+ place of an index lookup). A full ``ANALYZE``, not a sampled one: every
+ index leads with ``repository_id``, which a sample (``analysis_limit``, and
+ ``PRAGMA optimize`` since SQLite 3.46) reads as selective, so the planner
+ keeps the wrong index. It runs only when the statistics are missing or a
+ table's size has drifted, since a full pass reads the whole store. Other
+ backends keep their own statistics. A failure costs only a slower plan, so
+ it never fails the write.
+ """
+ if engine.dialect.name != "sqlite":
+ return
+ try:
+ async with engine.begin() as conn:
+ if await conn.run_sync(_planner_stats_stale):
+ await conn.execute(text("ANALYZE"))
+ except DBAPIError as exc: # a locked or read-only store
+ log.warning("sqlite.analyze_failed", error=str(exc))
+
+
async def schema_behind(engine: AsyncEngine) -> bool:
"""Whether a table or column the models declare is missing from the store.
diff --git a/packages/core/src/repowise/core/pipeline/full_index.py b/packages/core/src/repowise/core/pipeline/full_index.py
index d0091df84..55d139048 100644
--- a/packages/core/src/repowise/core/pipeline/full_index.py
+++ b/packages/core/src/repowise/core/pipeline/full_index.py
@@ -38,6 +38,7 @@ async def index_repo_full(
create_session_factory,
get_session,
init_db,
+ refresh_planner_stats,
upsert_repository,
)
from repowise.core.persistence.database import resolve_db_url
@@ -90,6 +91,7 @@ async def index_repo_full(
from repowise.core.persistence.crud.analysis.actions import write_read_snapshots
await write_read_snapshots(session, repo.id)
+ await refresh_planner_stats(engine)
from repowise.core.pipeline.cleanup_debt import (
clear_cleanup_debt,
diff --git a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
index 951b9813a..3d47bd0bb 100644
--- a/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
+++ b/packages/core/src/repowise/core/pipeline/incremental/persist_index.py
@@ -603,6 +603,7 @@ async def _write(ctx: IncrementalPersist) -> None:
create_session_factory,
get_session,
init_db,
+ refresh_planner_stats,
upsert_repository,
)
from repowise.core.persistence.database import resolve_db_url
@@ -649,6 +650,8 @@ async def _write(ctx: IncrementalPersist) -> None:
set(ctx.tombstoned_page_ids) | set(ctx.deselected_page_ids) | cleanup_debt["vectors"]
)
await run_steps(ctx, CLEANUP_STEPS)
+ with timed(timings, "persist.planner_stats"):
+ await refresh_planner_stats(ctx.engine)
# Ceiling: retired non-file pages' *vector* embeddings survive this path.
# There is no store here to delete them from, and building one would
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index 7cabd8e0f..0d267fe7e 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -29,7 +29,7 @@ from repowise.core.persistence.crud import (
get_repository,
update_job_status,
)
-from repowise.core.persistence.database import get_session
+from repowise.core.persistence.database import get_session, refresh_planner_stats
from repowise.core.pipeline import persist_pipeline_result, run_pipeline
from repowise.core.pipeline.modes import OrchestratorMode
from repowise.server.job_events import JobEventBuffer, create_event_buffer
@@ -565,6 +565,7 @@ async def _run_index_job(run: _JobRun, app_state: Any) -> None:
all_pages = (result.generated_pages or []) + incremental_pages
await _refresh_fts(run.fts, swept_page_ids, all_pages)
+ await refresh_planner_stats(run.session_factory.kw["bind"])
# Stop progress updates first, so none overwrites "completed".
await run.progress.drain_and_stop()
diff --git a/tests/unit/persistence/test_database_pragmas.py b/tests/unit/persistence/test_database_pragmas.py
index 9e6bc9b13..4884410ee 100644
--- a/tests/unit/persistence/test_database_pragmas.py
+++ b/tests/unit/persistence/test_database_pragmas.py
@@ -285,3 +285,103 @@ def test_a_non_operational_pragma_failure_still_raises() -> None:
with pytest.raises(sqlite3.ProgrammingError):
listener(_Conn(), None)
+
+
+async def test_planner_stats_count_every_row_of_a_single_repository(tmp_path: Path) -> None:
+ """Every index leads with ``repository_id``. A sampled ANALYZE reads that
+ column as selective and the planner then scans the wide covering index for
+ a target lookup, so the stats must come from the whole table."""
+ from repowise.core.persistence import create_session_factory, refresh_planner_stats
+ from repowise.core.persistence.models import GraphEdge, Repository
+
+ db = tmp_path / "wiki.db"
+ engine = create_engine(f"sqlite+aiosqlite:///{db}")
+ try:
+ await init_db(engine)
+ async with create_session_factory(engine)() as session:
+ session.add(Repository(id="r1", name="r", local_path="/r"))
+ await session.flush()
+ session.add_all(
+ GraphEdge(repository_id="r1", source_node_id=f"s{i % 3}", target_node_id=f"t{i}")
+ for i in range(3000)
+ )
+ await session.commit()
+ await refresh_planner_stats(engine)
+ finally:
+ await engine.dispose()
+
+ conn = sqlite3.connect(db)
+ stat = conn.execute(
+ "SELECT stat FROM sqlite_stat1 WHERE idx = 'ix_graph_edges_repo_target'"
+ ).fetchone()
+ plan = conn.execute(
+ "EXPLAIN QUERY PLAN SELECT DISTINCT source_node_id, target_node_id FROM graph_edges "
+ "WHERE repository_id = 'r1' AND target_node_id IN ('t1', 't2') AND edge_type IN ('imports')"
+ ).fetchall()
+ conn.close()
+ assert stat is not None and stat[0].split()[:2] == ["3000", "3000"]
+ assert "ix_graph_edges_repo_target" in str(plan)
+
+
+async def test_planner_stats_rerun_only_when_a_table_drifts(tmp_path: Path) -> None:
+ """A full pass reads the whole store, so a write that leaves every table
+ near the size ANALYZE recorded keeps the statistics it has."""
+ from repowise.core.persistence import create_session_factory, refresh_planner_stats
+ from repowise.core.persistence.models import GraphEdge, Repository
+
+ db = tmp_path / "wiki.db"
+ engine = create_engine(f"sqlite+aiosqlite:///{db}")
+
+ async def add_edges(start: int, count: int) -> None:
+ async with create_session_factory(engine)() as session:
+ session.add_all(
+ GraphEdge(repository_id="r1", source_node_id="s", target_node_id=f"t{i}")
+ for i in range(start, start + count)
+ )
+ await session.commit()
+
+ def recorded() -> str:
+ conn = sqlite3.connect(db)
+ try:
+ return conn.execute(
+ "SELECT stat FROM sqlite_stat1 WHERE idx = 'ix_graph_edges_repo_target'"
+ ).fetchone()[0]
+ finally:
+ conn.close()
+
+ try:
+ await init_db(engine)
+ async with create_session_factory(engine)() as session:
+ session.add(Repository(id="r1", name="r", local_path="/r"))
+ await session.commit()
+ await add_edges(0, 100)
+ await refresh_planner_stats(engine)
+ await add_edges(100, 50)
+ await refresh_planner_stats(engine)
+ within_band = recorded()
+ await add_edges(150, 100)
+ await refresh_planner_stats(engine)
+ drifted = recorded()
+ finally:
+ await engine.dispose()
+
+ assert within_band.split()[0] == "100"
+ assert drifted.split()[0] == "250"
+
+
+async def test_planner_stats_skip_other_backends() -> None:
+ from repowise.core.persistence import refresh_planner_stats
+
+ class _Dialect:
+ name = "postgresql"
+
+ class _Engine:
+ dialect = _Dialect()
+
+ def begin(self): # pragma: no cover - reaching it is the failure
+ raise AssertionError("ANALYZE must not run off SQLite")
+
+ await refresh_planner_stats(_Engine()) # type: ignore[arg-type]