Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(persistence): stop the WAL switch killing a read against a live writer (#2064)
Raghav Chamadiya228da8888e69
diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.pyindex d47b1e7cc..605b159a7 100644--- a/packages/core/src/repowise/core/persistence/database.py+++ b/packages/core/src/repowise/core/persistence/database.py@@ -12,6 +12,7 @@ Call init_db() once at startup to create all tables and the FTS index.from __future__ import annotationsimport os+import sqlite3from collections.abc import AsyncGenerator, Callablefrom contextlib import asynccontextmanagerfrom pathlib import Path@@ -43,13 +44,17 @@ log = structlog.get_logger(__name__)# for large repos. SQLite blocks (doesn't busy-loop) so this is cheap._SQLITE_BUSY_TIMEOUT_MS = 30000-def _sqlite_pragmas(busy_timeout_ms: int) -> tuple[tuple[str, str], ...]:- """Return the pragma list to apply to a SQLite connection."""+ """Return the pragma list to apply to a SQLite connection.++ ``busy_timeout`` leads so that every pragma and statement after it on this+ connection inherits the retry window, rather than the window arriving only+ once the connection is most of the way set up.+ """return (+ ("busy_timeout", str(busy_timeout_ms)),("journal_mode", "WAL"),("synchronous", "NORMAL"),- ("busy_timeout", str(busy_timeout_ms)),("foreign_keys", "ON"),)@@ -65,8 +70,59 @@ def _make_pragma_listener(busy_timeout_ms: int):(issue #326)."""+ # Log the first failed switch per engine, not every connection: this engine+ # uses NullPool, so it reconnects per checkout and a store that cannot take+ # WAL would otherwise emit a warning per query. Behaviour does not depend on+ # this flag - every connection still attempts the switch.+ warned = False++ def _set_journal_mode_wal(cursor: object) -> None:+ """Re-issue the WAL switch, but never let it fail the connection.++ The re-issue is defensive: WAL persists in the file, so a store this+ repowise created is already in WAL and the pragma is a no-op that+ cannot contend. It exists only for a store written by an older+ repowise, by ``alembic``, or on a filesystem that refused the first+ switch - and there it can legitimately fail:++ * a concurrent writer holds the brief exclusive lock the transition+ needs, and SQLite does NOT route that lock through the busy handler,+ so it returns SQLITE_BUSY immediately however large ``busy_timeout``+ is;+ * the store or its directory is read-only, giving "attempt to write a+ readonly database".++ None of those stop the connection being useful. The store keeps the+ journal mode it already has and every query still runs, so the failure+ is swallowed here and left to surface on a statement that actually+ needs the write. What shipped before raised out of the ``connect``+ event and took out the whole connection at open time, including reads+ that would have succeeded.++ Deliberately not retried. The listener runs on the event-loop thread+ under SQLAlchemy's greenlet bridge, so sleeping here blocks the loop -+ and when the writer is another task in this same process, that stops it+ reaching the COMMIT the retry is waiting on. The next connection tries+ again at no cost instead.+ """+ nonlocal warned+ try:+ # journal_mode returns the new mode and must be queried, not+ # assigned, because in :memory: databases it silently+ # downgrades to MEMORY.+ cursor.execute("PRAGMA journal_mode=WAL") # type: ignore[attr-defined]+ except sqlite3.OperationalError as exc:+ if not warned:+ warned = True+ log.warning(+ "sqlite: could not switch the store to WAL (%s). Continuing "+ "in its current journal mode; concurrent reads may block on "+ "writes until a later connection switches it.",+ exc,+ )+def _apply_sqlite_pragmas(dbapi_connection: object, _connection_record: object) -> None:- """Apply WAL, busy_timeout, and FK pragmas on every new SQLite connection.+ """Apply busy_timeout, WAL, and FK pragmas on every new SQLite connection.Registered as a ``connect`` event listener so it runs once per physicalconnection, including the first one opened after the engine is created and@@ -78,8 +134,9 @@ def _make_pragma_listener(busy_timeout_ms: int):cursor = dbapi_connection.cursor() # type: ignore[attr-defined]try:for name, value in _sqlite_pragmas(busy_timeout_ms):- # journal_mode returns the new mode and must be queried, not assigned,- # because in :memory: databases it silently downgrades to MEMORY.+ if name == "journal_mode":+ _set_journal_mode_wal(cursor)+ continuecursor.execute(f"PRAGMA {name}={value}")finally:cursor.close()diff --git a/tests/unit/persistence/test_database_pragmas.py b/tests/unit/persistence/test_database_pragmas.pyindex 57a7cf279..223a41104 100644--- a/tests/unit/persistence/test_database_pragmas.py+++ b/tests/unit/persistence/test_database_pragmas.py@@ -13,6 +13,10 @@ import pytestfrom sqlalchemy import textfrom repowise.core.persistence import create_engine, init_db+from repowise.core.persistence.database import (+ _make_pragma_listener,+ _sqlite_pragmas,+)async def _read_pragma(engine, pragma: str) -> str:@@ -140,3 +144,133 @@ def test_pragmas_survive_a_fresh_sync_open(tmp_path: Path) -> None:assert mode.lower() == "wal"finally:conn.close()+++def _seed_non_wal_db(db_path: Path) -> None:+ """Create a store left in ``delete`` journal mode, as an older repowise, a+ third-party tool, or a filesystem that refused the first WAL switch would."""+ conn = sqlite3.connect(db_path)+ try:+ conn.execute("PRAGMA journal_mode=DELETE")+ conn.execute("CREATE TABLE t (k INTEGER PRIMARY KEY, v INTEGER)")+ conn.execute("INSERT INTO t (k, v) VALUES (1, 0)")+ conn.commit()+ finally:+ conn.close()+++async def test_read_survives_a_writer_holding_a_non_wal_store(tmp_path: Path) -> None:+ """A reader must not die at connect time because the WAL switch lost a race.++ SQLite does not route the ``journal_mode`` change through the busy handler:+ against a held write transaction it returns SQLITE_BUSY immediately, whatever+ ``busy_timeout`` says. The pragma is a no-op on a store already in WAL, so+ only a store still in ``delete`` reaches this — and there the old listener+ raised ``OperationalError: database is locked`` out of the ``connect`` event,+ killing the connection before any query ran, including reads that would have+ succeeded. This is the MCP server's ``OperationalError`` (issue #2059).+ """+ db_path = tmp_path / "wiki.db"+ _seed_non_wal_db(db_path)++ writer = sqlite3.connect(db_path, timeout=30)+ writer.execute("PRAGMA busy_timeout=30000")+ writer.execute("BEGIN IMMEDIATE")+ writer.execute("UPDATE t SET v = 99 WHERE k = 1")+ try:+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")+ try:+ async with engine.connect() as conn:+ rows = (await conn.execute(text("SELECT v FROM t WHERE k = 1"))).fetchall()+ assert rows == [(0,)]+ finally:+ await engine.dispose()+ finally:+ writer.rollback()+ writer.close()+++async def test_wal_switch_still_applies_once_the_writer_releases(tmp_path: Path) -> None:+ """Tolerating the busy switch must not mean abandoning WAL. Once no writer+ holds the store, the next connection upgrades it as before."""+ db_path = tmp_path / "wiki.db"+ _seed_non_wal_db(db_path)++ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")+ try:+ writer = sqlite3.connect(db_path, timeout=30)+ writer.execute("PRAGMA busy_timeout=30000")+ writer.execute("BEGIN IMMEDIATE")+ writer.execute("UPDATE t SET v = 99 WHERE k = 1")+ try:+ async with engine.connect() as conn:+ await conn.execute(text("SELECT v FROM t WHERE k = 1"))+ finally:+ writer.rollback()+ writer.close()++ assert (await _read_pragma(engine, "journal_mode")).lower() == "wal"+ finally:+ await engine.dispose()+++def test_busy_timeout_is_the_first_pragma_applied() -> None:+ """Everything after ``busy_timeout`` on a connection inherits its retry+ window, so it has to lead the list rather than arrive third."""+ assert _sqlite_pragmas(30000)[0][0] == "busy_timeout"+++def _listener_with_journal_mode_error(exc: Exception):+ """Drive the connect listener against a cursor whose WAL switch fails."""+ listener = _make_pragma_listener(30000)+ applied: list[str] = []++ class _Cursor:+ def execute(self, sql: str) -> None:+ if "journal_mode" in sql:+ raise exc+ applied.append(sql)++ def close(self) -> None:+ return None++ class _Conn:+ def cursor(self) -> _Cursor:+ return _Cursor()++ listener(_Conn(), None)+ return applied+++def test_a_readonly_store_still_gets_a_usable_connection() -> None:+ """The other way the defensive re-issue fails: the store or its directory is+ read-only. Reads from it are still perfectly good, so this must not raise+ either - and the pragmas after it must still be applied."""+ applied = _listener_with_journal_mode_error(+ sqlite3.OperationalError("attempt to write a readonly database")+ )+ assert any("foreign_keys" in sql for sql in applied)+++def test_a_non_operational_pragma_failure_still_raises() -> None:+ """Tolerance is scoped to what the defensive re-issue can legitimately hit.+ A failure that is not an ``OperationalError`` is not that, and must surface.+ """+ listener = _make_pragma_listener(30000)++ class _Cursor:+ def execute(self, sql: str) -> None:+ if "journal_mode" in sql:+ raise sqlite3.ProgrammingError("cannot operate on a closed database")++ def close(self) -> None:+ return None++ class _Conn:+ def cursor(self) -> _Cursor:+ return _Cursor()++ with pytest.raises(sqlite3.ProgrammingError):+ listener(_Conn(), None)