Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(db): retry only SQLite's busy error in retry_on_db_lock (#7753)

Lukasaa8de47dc393
diff --git a/changelog.d/7753.bugfix.md b/changelog.d/7753.bugfix.md
new file mode 100644
index 0000000000..63fc3fbf0e
--- /dev/null
+++ b/changelog.d/7753.bugfix.md
@@ -0,0 +1 @@
+The shared `retry_on_db_lock` helper now retries only SQLite's busy error. It wraps the writes, resets and cleanups of rate-limit estimates, the token metrics writes, and the deleting of a research and the clearing of the research history. In these operations, a `database table is locked` (SQLITE_LOCKED) error, or another error whose SQL statement mentions `locked`, now fails at once instead of being retried. A busy error is recognized by its SQLite error code when the driver sets one, so a reworded message still retries.
diff --git a/src/local_deep_research/database/sqlcipher_utils.py b/src/local_deep_research/database/sqlcipher_utils.py
index c63bfe5b9a..79697bc54f 100644
--- a/src/local_deep_research/database/sqlcipher_utils.py
+++ b/src/local_deep_research/database/sqlcipher_utils.py
@@ -833,11 +833,10 @@ def _db_lock_error_types() -> Tuple[Type[BaseException], ...]:
SQLAlchemy session, and SQLAlchemy wraps the DBAPI lock error in
its own ``OperationalError`` subclass that does NOT inherit from
either DBAPI class -- without it the helper would silently be a
- no-op on the wrapped path and the retry would never fire. The
- ``"locked" in str(exc)`` message guard below filters the SQLAlchemy
- subclass correctly: on a ``SQLITE_BUSY`` the wrapped message
- preserves the substring; on every other ``OperationalError``
- (broken pipe, schema error, etc.) it does not.
+ no-op on the wrapped path and the retry would never fire.
+ ``_is_database_busy`` then reads the DBAPI error the SQLAlchemy
+ subclass wraps, by its ``sqlite_errorcode`` when the driver sets one
+ and by its message otherwise, so only ``SQLITE_BUSY`` is retried.
"""
types: list[Type[BaseException]] = [sqlite3.OperationalError]
try:
@@ -858,19 +857,42 @@ def _db_lock_error_types() -> Tuple[Type[BaseException], ...]:
_DB_LOCK_ERROR_TYPES = _db_lock_error_types()
+def _is_database_busy(exc: BaseException) -> bool:
+ """Whether ``exc`` is SQLite's busy error, ``database is locked``.
+
+ ``str()`` of a SQLAlchemy ``OperationalError`` also renders the SQL
+ statement and its parameters, so the DBAPI error it wraps
+ (``exc.orig``) is read instead. When the driver sets
+ ``sqlite_errorcode``, the code decides: ``SQLITE_BUSY`` or one of its
+ extended codes is busy, whatever the wording of the message. Without a
+ code, the message must be exactly ``database is locked``.
+ ``database table is locked`` (SQLITE_LOCKED) is not a busy-timeout
+ condition and matches neither.
+ """
+ original = getattr(exc, "orig", None) or exc
+ code = getattr(original, "sqlite_errorcode", None)
+ if isinstance(code, int) and not isinstance(code, bool):
+ return code & 0xFF == sqlite3.SQLITE_BUSY
+ return str(original).strip().lower() == "database is locked"
+
+
def retry_on_db_lock(
callable_: Callable[[], _T],
*,
attempts: int = 3,
base_delay_seconds: float = 0.1,
) -> _T:
- """Run ``callable_``, retrying on SQLite ``database is locked`` errors.
-
- Only retries ``OperationalError`` whose message contains
- ``"locked"`` -- every other exception (connection errors,
- IntegrityError, ProgrammingError, plain ``OperationalError`` without
- ``"locked"`` in the message, etc.) propagates immediately so the
- caller can decide how to handle it.
+ """Run ``callable_``, retrying on SQLite's busy error.
+
+ Only retries an ``OperationalError`` that ``_is_database_busy`` reads
+ as SQLITE_BUSY: by the DBAPI error's ``sqlite_errorcode`` when the
+ driver sets one (``SQLITE_BUSY`` or one of its extended codes), and
+ otherwise by a message of exactly ``database is locked``. Every other
+ exception (connection errors, IntegrityError, ProgrammingError, any
+ other ``OperationalError``, including SQLITE_LOCKED's ``database table
+ is locked`` and an error whose SQL or parameters happen to contain
+ ``locked``) propagates immediately so the caller can decide how to
+ handle it.
The total wall-clock budget before the helper gives up is bounded
by ``attempts * busy_timeout + (attempts - 1) * base_delay_seconds *
@@ -912,14 +934,14 @@ def retry_on_db_lock(
sqlite3.OperationalError / sqlcipher3.OperationalError /
sqlalchemy.exc.OperationalError: ``database is locked`` if
every attempt is exhausted. Other exceptions propagate
- without retry (the message-substring guard filters the
- SQLAlchemy wrapper to lock errors specifically).
+ without retry (``_is_database_busy`` filters the SQLAlchemy
+ wrapper to lock errors specifically).
"""
for attempt in range(1, attempts + 1):
try:
return callable_()
except _DB_LOCK_ERROR_TYPES as exc:
- if "locked" not in str(exc).lower() or attempt >= attempts:
+ if not _is_database_busy(exc) or attempt >= attempts:
raise
logger.debug(
"SQLite 'database is locked' on attempt "
diff --git a/tests/database/test_retry_on_db_lock.py b/tests/database/test_retry_on_db_lock.py
index d361d9509d..bbb277f317 100644
--- a/tests/database/test_retry_on_db_lock.py
+++ b/tests/database/test_retry_on_db_lock.py
@@ -2,10 +2,11 @@
The helper is the production answer to the rate-limit tracker's
cascading ``database is locked`` failures (2026-09-21). It retries the
-wrapped callable on ``OperationalError`` whose message contains
-``"locked"``, and only on that exception class -- every other failure
-(IntegrityError, ProgrammingError, plain ConnectionError, etc.)
-propagates immediately so the caller can decide how to handle it.
+wrapped callable on ``OperationalError`` whose DBAPI message is exactly
+``database is locked``, and only on that exception class -- every other
+failure (IntegrityError, ProgrammingError, plain ConnectionError,
+SQLITE_LOCKED, etc.) propagates immediately so the caller can decide
+how to handle it.
These tests pin the contract:
@@ -93,6 +94,149 @@ class TestRetryOnLockContract:
retry_on_db_lock(raises_other_op_error)
assert calls == [1]
+ def test_does_not_retry_on_sqlite_locked(self):
+ """``database table is locked`` is SQLITE_LOCKED, not the busy
+ timeout the helper waits out, so it propagates on the first
+ attempt.
+ """
+ calls = []
+
+ def raises_table_locked():
+ calls.append(1)
+ raise sqlite3.OperationalError("database table is locked")
+
+ with pytest.raises(sqlite3.OperationalError, match="table is locked"):
+ retry_on_db_lock(raises_table_locked, base_delay_seconds=0)
+ assert calls == [1]
+
+ def test_does_not_retry_when_only_the_sql_mentions_locked(self):
+ """``str()`` of a SQLAlchemy ``OperationalError`` renders the
+ statement and its parameters. A non-lock error whose query or
+ parameters mention ``locked`` must still propagate on the first
+ attempt.
+ """
+ calls = []
+
+ def raises_wrapped_error():
+ calls.append(1)
+ raise SAOperationalError(
+ "UPDATE accounts SET status = ? WHERE note = 'locked'",
+ ("locked",),
+ sqlite3.OperationalError("no such table: accounts"),
+ )
+
+ with pytest.raises(SAOperationalError, match="no such table"):
+ retry_on_db_lock(raises_wrapped_error, base_delay_seconds=0)
+ assert calls == [1]
+
+ def test_retries_a_wrapped_busy_error(self):
+ """The SQLAlchemy wrapper around a real ``database is locked`` is
+ retried: the helper reads the DBAPI message, not the rendered
+ statement.
+ """
+ calls = []
+
+ def fails_then_succeeds():
+ calls.append(1)
+ if len(calls) < 2:
+ raise SAOperationalError(
+ "UPDATE accounts SET status = ?",
+ ("active",),
+ sqlite3.OperationalError("database is locked"),
+ )
+ return "ok"
+
+ assert (
+ retry_on_db_lock(fails_then_succeeds, base_delay_seconds=0) == "ok"
+ )
+ assert len(calls) == 2
+
+ @staticmethod
+ def _busy_error(message, code=None):
+ error = sqlite3.OperationalError(message)
+ if code is not None:
+ error.sqlite_errorcode = code
+ return error
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ sqlite3.SQLITE_BUSY,
+ sqlite3.SQLITE_BUSY_RECOVERY,
+ sqlite3.SQLITE_BUSY_SNAPSHOT,
+ ],
+ )
+ def test_retries_on_a_busy_error_code_whatever_the_message(self, code):
+ """With ``sqlite_errorcode`` set, the code decides, so a busy
+ error still retries if a driver words its message differently.
+ """
+ calls = []
+
+ def fails_then_succeeds():
+ calls.append(1)
+ if len(calls) < 2:
+ raise self._busy_error(
+ "database is locked (5) (SQLITE_BUSY)", code
+ )
+ return "ok"
+
+ assert (
+ retry_on_db_lock(fails_then_succeeds, base_delay_seconds=0) == "ok"
+ )
+ assert len(calls) == 2
+
+ def test_does_not_retry_on_a_non_busy_error_code(self):
+ """A code other than a busy one is not retried, even with the
+ busy message.
+ """
+ calls = []
+
+ def raises_locked():
+ calls.append(1)
+ raise self._busy_error("database is locked", sqlite3.SQLITE_LOCKED)
+
+ with pytest.raises(sqlite3.OperationalError):
+ retry_on_db_lock(raises_locked, base_delay_seconds=0)
+ assert calls == [1]
+
+ @pytest.mark.parametrize(
+ "message", [" database is locked ", "Database Is Locked"]
+ )
+ def test_matches_busy_message_case_and_whitespace_insensitive(
+ self, message
+ ):
+ calls = []
+
+ def fails_then_succeeds():
+ calls.append(1)
+ if len(calls) < 2:
+ raise sqlite3.OperationalError(message)
+ return "ok"
+
+ assert (
+ retry_on_db_lock(fails_then_succeeds, base_delay_seconds=0) == "ok"
+ )
+ assert len(calls) == 2
+
+ def test_falls_back_to_the_wrapped_error_when_orig_is_falsey(self):
+ """An error whose ``orig`` is falsey is read itself."""
+
+ class BusyWithoutOrig(sqlite3.OperationalError):
+ orig = None
+
+ calls = []
+
+ def fails_then_succeeds():
+ calls.append(1)
+ if len(calls) < 2:
+ raise BusyWithoutOrig("database is locked")
+ return "ok"
+
+ assert (
+ retry_on_db_lock(fails_then_succeeds, base_delay_seconds=0) == "ok"
+ )
+ assert len(calls) == 2
+
def test_retries_until_success(self):
"""The callable fails with a lock error N-1 times, then
succeeds on the Nth attempt -- the helper returns the success