Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(persistence): stop the WAL switch killing a read against a live writer (#2064)

Raghav Chamadiya228da8888e69
diff --git a/packages/core/src/repowise/core/persistence/database.py b/packages/core/src/repowise/core/persistence/database.py
index d47b1e7cc..605b159a7 100644
--- a/packages/core/src/repowise/core/persistence/database.py
+++ b/packages/core/src/repowise/core/persistence/database.py
@@ -12,6 +12,7 @@ Call init_db() once at startup to create all tables and the FTS index.
from __future__ import annotations
import os
+import sqlite3
from collections.abc import AsyncGenerator, Callable
from contextlib import asynccontextmanager
from pathlib import Path
@@ -43,13 +44,17 @@ log = structlog.get_logger(__name__)
# for large repos. SQLite blocks (doesn't busy-loop) so this is cheap.
_SQLITE_BUSY_TIMEOUT_MS = 30000
-
def _sqlite_pragmas(busy_timeout_ms: int) -> tuple[tuple[str, str], ...]:
- """Return the pragma list to apply to a SQLite connection."""
+ """Return the pragma list to apply to a SQLite connection.
+
+ ``busy_timeout`` leads so that every pragma and statement after it on this
+ connection inherits the retry window, rather than the window arriving only
+ once the connection is most of the way set up.
+ """
return (
+ ("busy_timeout", str(busy_timeout_ms)),
("journal_mode", "WAL"),
("synchronous", "NORMAL"),
- ("busy_timeout", str(busy_timeout_ms)),
("foreign_keys", "ON"),
)
@@ -65,8 +70,59 @@ def _make_pragma_listener(busy_timeout_ms: int):
(issue #326).
"""
+ # Log the first failed switch per engine, not every connection: this engine
+ # uses NullPool, so it reconnects per checkout and a store that cannot take
+ # WAL would otherwise emit a warning per query. Behaviour does not depend on
+ # this flag - every connection still attempts the switch.
+ warned = False
+
+ def _set_journal_mode_wal(cursor: object) -> None:
+ """Re-issue the WAL switch, but never let it fail the connection.
+
+ The re-issue is defensive: WAL persists in the file, so a store this
+ repowise created is already in WAL and the pragma is a no-op that
+ cannot contend. It exists only for a store written by an older
+ repowise, by ``alembic``, or on a filesystem that refused the first
+ switch - and there it can legitimately fail:
+
+ * a concurrent writer holds the brief exclusive lock the transition
+ needs, and SQLite does NOT route that lock through the busy handler,
+ so it returns SQLITE_BUSY immediately however large ``busy_timeout``
+ is;
+ * the store or its directory is read-only, giving "attempt to write a
+ readonly database".
+
+ None of those stop the connection being useful. The store keeps the
+ journal mode it already has and every query still runs, so the failure
+ is swallowed here and left to surface on a statement that actually
+ needs the write. What shipped before raised out of the ``connect``
+ event and took out the whole connection at open time, including reads
+ that would have succeeded.
+
+ Deliberately not retried. The listener runs on the event-loop thread
+ under SQLAlchemy's greenlet bridge, so sleeping here blocks the loop -
+ and when the writer is another task in this same process, that stops it
+ reaching the COMMIT the retry is waiting on. The next connection tries
+ again at no cost instead.
+ """
+ nonlocal warned
+ try:
+ # journal_mode returns the new mode and must be queried, not
+ # assigned, because in :memory: databases it silently
+ # downgrades to MEMORY.
+ cursor.execute("PRAGMA journal_mode=WAL") # type: ignore[attr-defined]
+ except sqlite3.OperationalError as exc:
+ if not warned:
+ warned = True
+ log.warning(
+ "sqlite: could not switch the store to WAL (%s). Continuing "
+ "in its current journal mode; concurrent reads may block on "
+ "writes until a later connection switches it.",
+ exc,
+ )
+
def _apply_sqlite_pragmas(dbapi_connection: object, _connection_record: object) -> None:
- """Apply WAL, busy_timeout, and FK pragmas on every new SQLite connection.
+ """Apply busy_timeout, WAL, and FK pragmas on every new SQLite connection.
Registered as a ``connect`` event listener so it runs once per physical
connection, including the first one opened after the engine is created and
@@ -78,8 +134,9 @@ def _make_pragma_listener(busy_timeout_ms: int):
cursor = dbapi_connection.cursor() # type: ignore[attr-defined]
try:
for name, value in _sqlite_pragmas(busy_timeout_ms):
- # journal_mode returns the new mode and must be queried, not assigned,
- # because in :memory: databases it silently downgrades to MEMORY.
+ if name == "journal_mode":
+ _set_journal_mode_wal(cursor)
+ continue
cursor.execute(f"PRAGMA {name}={value}")
finally:
cursor.close()
diff --git a/tests/unit/persistence/test_database_pragmas.py b/tests/unit/persistence/test_database_pragmas.py
index 57a7cf279..223a41104 100644
--- a/tests/unit/persistence/test_database_pragmas.py
+++ b/tests/unit/persistence/test_database_pragmas.py
@@ -13,6 +13,10 @@ import pytest
from sqlalchemy import text
from repowise.core.persistence import create_engine, init_db
+from repowise.core.persistence.database import (
+ _make_pragma_listener,
+ _sqlite_pragmas,
+)
async def _read_pragma(engine, pragma: str) -> str:
@@ -140,3 +144,133 @@ def test_pragmas_survive_a_fresh_sync_open(tmp_path: Path) -> None:
assert mode.lower() == "wal"
finally:
conn.close()
+
+
+def _seed_non_wal_db(db_path: Path) -> None:
+ """Create a store left in ``delete`` journal mode, as an older repowise, a
+ third-party tool, or a filesystem that refused the first WAL switch would."""
+ conn = sqlite3.connect(db_path)
+ try:
+ conn.execute("PRAGMA journal_mode=DELETE")
+ conn.execute("CREATE TABLE t (k INTEGER PRIMARY KEY, v INTEGER)")
+ conn.execute("INSERT INTO t (k, v) VALUES (1, 0)")
+ conn.commit()
+ finally:
+ conn.close()
+
+
+async def test_read_survives_a_writer_holding_a_non_wal_store(tmp_path: Path) -> None:
+ """A reader must not die at connect time because the WAL switch lost a race.
+
+ SQLite does not route the ``journal_mode`` change through the busy handler:
+ against a held write transaction it returns SQLITE_BUSY immediately, whatever
+ ``busy_timeout`` says. The pragma is a no-op on a store already in WAL, so
+ only a store still in ``delete`` reaches this — and there the old listener
+ raised ``OperationalError: database is locked`` out of the ``connect`` event,
+ killing the connection before any query ran, including reads that would have
+ succeeded. This is the MCP server's ``OperationalError`` (issue #2059).
+ """
+ db_path = tmp_path / "wiki.db"
+ _seed_non_wal_db(db_path)
+
+ writer = sqlite3.connect(db_path, timeout=30)
+ writer.execute("PRAGMA busy_timeout=30000")
+ writer.execute("BEGIN IMMEDIATE")
+ writer.execute("UPDATE t SET v = 99 WHERE k = 1")
+ try:
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ try:
+ async with engine.connect() as conn:
+ rows = (await conn.execute(text("SELECT v FROM t WHERE k = 1"))).fetchall()
+ assert rows == [(0,)]
+ finally:
+ await engine.dispose()
+ finally:
+ writer.rollback()
+ writer.close()
+
+
+async def test_wal_switch_still_applies_once_the_writer_releases(tmp_path: Path) -> None:
+ """Tolerating the busy switch must not mean abandoning WAL. Once no writer
+ holds the store, the next connection upgrades it as before."""
+ db_path = tmp_path / "wiki.db"
+ _seed_non_wal_db(db_path)
+
+ engine = create_engine(f"sqlite+aiosqlite:///{db_path}")
+ try:
+ writer = sqlite3.connect(db_path, timeout=30)
+ writer.execute("PRAGMA busy_timeout=30000")
+ writer.execute("BEGIN IMMEDIATE")
+ writer.execute("UPDATE t SET v = 99 WHERE k = 1")
+ try:
+ async with engine.connect() as conn:
+ await conn.execute(text("SELECT v FROM t WHERE k = 1"))
+ finally:
+ writer.rollback()
+ writer.close()
+
+ assert (await _read_pragma(engine, "journal_mode")).lower() == "wal"
+ finally:
+ await engine.dispose()
+
+
+def test_busy_timeout_is_the_first_pragma_applied() -> None:
+ """Everything after ``busy_timeout`` on a connection inherits its retry
+ window, so it has to lead the list rather than arrive third."""
+ assert _sqlite_pragmas(30000)[0][0] == "busy_timeout"
+
+
+def _listener_with_journal_mode_error(exc: Exception):
+ """Drive the connect listener against a cursor whose WAL switch fails."""
+ listener = _make_pragma_listener(30000)
+ applied: list[str] = []
+
+ class _Cursor:
+ def execute(self, sql: str) -> None:
+ if "journal_mode" in sql:
+ raise exc
+ applied.append(sql)
+
+ def close(self) -> None:
+ return None
+
+ class _Conn:
+ def cursor(self) -> _Cursor:
+ return _Cursor()
+
+ listener(_Conn(), None)
+ return applied
+
+
+def test_a_readonly_store_still_gets_a_usable_connection() -> None:
+ """The other way the defensive re-issue fails: the store or its directory is
+ read-only. Reads from it are still perfectly good, so this must not raise
+ either - and the pragmas after it must still be applied."""
+ applied = _listener_with_journal_mode_error(
+ sqlite3.OperationalError("attempt to write a readonly database")
+ )
+ assert any("foreign_keys" in sql for sql in applied)
+
+
+def test_a_non_operational_pragma_failure_still_raises() -> None:
+ """Tolerance is scoped to what the defensive re-issue can legitimately hit.
+ A failure that is not an ``OperationalError`` is not that, and must surface.
+ """
+ listener = _make_pragma_listener(30000)
+
+ class _Cursor:
+ def execute(self, sql: str) -> None:
+ if "journal_mode" in sql:
+ raise sqlite3.ProgrammingError("cannot operate on a closed database")
+
+ def close(self) -> None:
+ return None
+
+ class _Conn:
+ def cursor(self) -> _Cursor:
+ return _Cursor()
+
+ with pytest.raises(sqlite3.ProgrammingError):
+ listener(_Conn(), None)