Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(backup): keep superseded-key purge going past a backup that raises (#7748)

Imaduddin Bhaisaheb82e867b5e8b6
diff --git a/changelog.d/7678.bugfix.md b/changelog.d/7678.bugfix.md
new file mode 100644
index 0000000000..d747ec44c7
--- /dev/null
+++ b/changelog.d/7678.bugfix.md
@@ -0,0 +1 @@
+The superseded-key backup purge now guarantees that a backup that cannot be read or classified is kept and never stops it from examining the remaining old-key backups.
diff --git a/src/local_deep_research/database/backup/backup_service.py b/src/local_deep_research/database/backup/backup_service.py
index 61e0467a4d..6b2e2ef5fc 100644
--- a/src/local_deep_research/database/backup/backup_service.py
+++ b/src/local_deep_research/database/backup/backup_service.py
@@ -8,6 +8,7 @@ import hmac
import os
import re
import shutil
+import sqlite3
import tempfile
import threading
import time
@@ -82,6 +83,23 @@ _QUARANTINED_BACKUP_NAME_RE = re.compile(
)
+# Errors one corrupt or unreadable backup can raise while it is classified,
+# moved aside or deleted. sqlcipher3's errors do not subclass sqlite3.Error,
+# so its DB-API base class is listed too.
+try:
+ from sqlcipher3 import dbapi2 as _sqlcipher_dbapi
+
+ _SQLCIPHER_ERRORS: tuple = (_sqlcipher_dbapi.Error,)
+except ImportError: # pragma: no cover - SQLCipher optional
+ _SQLCIPHER_ERRORS = ()
+_PURGE_ITEM_ERRORS = (
+ OSError,
+ sqlite3.Error,
+ ValueError,
+ *_SQLCIPHER_ERRORS,
+)
+
+
def _validate_backup_path(path: Path) -> str:
"""Normalize host separators and validate the path used in ATTACH.
@@ -1281,6 +1299,11 @@ class BackupService:
salt its database needs is left to the regular verification,
which already rejects it.
+ A backup that raises while it is examined (an I/O error, a
+ SQLite/SQLCipher error, an unreadable salt) is kept and logged;
+ the purge carries on with the remaining backups. Errors outside
+ ``_PURGE_ITEM_ERRORS`` are bugs and propagate.
+
Application callers must hold this database's rekey sidecar lock
and establish that this password still belongs to its current
stable epoch, through the entire purge. The per-user backup lock
@@ -1294,45 +1317,64 @@ class BackupService:
_get_backup_verifications(self.username).clear()
source_has_salt = has_per_database_salt(self.db_path)
for backup_path in self._managed_backup_paths():
- backup_salt = get_salt_file_path(backup_path)
- if (
- not backup_path.is_file()
- or backup_path.is_symlink()
- or backup_salt.is_symlink()
- or (source_has_salt and not backup_salt.exists())
- ):
- continue
- verdict = self._classify_backup_key(backup_path)
- if verdict == "damaged":
- if _QUARANTINED_BACKUP_NAME_RE.fullmatch(backup_path.name):
- continue
- moved = self._move_damaged_backup_aside(backup_path)
- if moved is None:
- logger.warning(
- f"Backup {backup_path.name} is under the current "
- "key but damaged, and could not be moved aside"
- )
- else:
- logger.warning(
- f"Backup {backup_path.name} is under the current "
- f"key but its first page is damaged; moved aside "
- f"as {moved.name}"
- )
- continue
- if verdict != "superseded":
- continue
+ # One unreadable or corrupt backup must not shield the rest
+ # from the purge: it is kept (fail-safe) and the loop moves
+ # on. Only the errors a bad backup can raise are caught;
+ # anything else is a bug and still propagates.
try:
- self._remove_backup_files(backup_path)
- deleted += 1
- logger.info(
- f"Purged backup under superseded key: {backup_path.name}"
- )
- except OSError:
+ if self._purge_one_superseded_backup(
+ backup_path, source_has_salt
+ ):
+ deleted += 1
+ except _PURGE_ITEM_ERRORS as error:
logger.warning(
- f"Could not delete backup {backup_path.name}"
+ f"Skipping backup {backup_path.name} in the "
+ f"superseded-key purge: {type(error).__name__}"
)
return deleted
+ def _purge_one_superseded_backup(
+ self, backup_path: Path, source_has_salt: bool
+ ) -> bool:
+ """Classify one backup and delete it or move it aside.
+
+ Returns True only when the backup was deleted as superseded.
+ """
+ backup_salt = get_salt_file_path(backup_path)
+ if (
+ not backup_path.is_file()
+ or backup_path.is_symlink()
+ or backup_salt.is_symlink()
+ or (source_has_salt and not backup_salt.exists())
+ ):
+ return False
+ verdict = self._classify_backup_key(backup_path)
+ if verdict == "damaged":
+ if _QUARANTINED_BACKUP_NAME_RE.fullmatch(backup_path.name):
+ return False
+ moved = self._move_damaged_backup_aside(backup_path)
+ if moved is None:
+ logger.warning(
+ f"Backup {backup_path.name} is under the current "
+ "key but damaged, and could not be moved aside"
+ )
+ else:
+ logger.warning(
+ f"Backup {backup_path.name} is under the current "
+ f"key but its first page is damaged; moved aside "
+ f"as {moved.name}"
+ )
+ return False
+ if verdict != "superseded":
+ return False
+ try:
+ self._remove_backup_files(backup_path)
+ except OSError:
+ logger.warning(f"Could not delete backup {backup_path.name}")
+ return False
+ logger.info(f"Purged backup under superseded key: {backup_path.name}")
+ return True
+
def get_latest_backup(self) -> Optional[Path]:
"""Get the path to the most recent backup.
diff --git a/tests/database/backup/test_superseded_key_purge.py b/tests/database/backup/test_superseded_key_purge.py
index e076278285..1935796653 100644
--- a/tests/database/backup/test_superseded_key_purge.py
+++ b/tests/database/backup/test_superseded_key_purge.py
@@ -10,6 +10,7 @@ SQLCipher throughout.
import multiprocessing
import os
+import sqlite3
import stat
import time
import uuid
@@ -39,6 +40,8 @@ from local_deep_research.database.sqlcipher_utils import (
set_sqlcipher_key,
)
+sqlcipher_dbapi = pytest.importorskip("sqlcipher3.dbapi2")
+
OLD = "OldPurgeHorse1!" # noqa: S105
NEW = "NewPurgeHorse2!" # noqa: S105
LATEST = "LatestPurgeHorse3!" # noqa: S105
@@ -323,6 +326,137 @@ def test_purge_moves_a_damaged_current_key_backup_aside(manager):
assert moved_db[0].exists()
+def _first_call_raises(error):
+ """Patch ``_classify_backup_key`` so its first call raises ``error``."""
+ real = BackupService._classify_backup_key
+ calls = []
+
+ def classify(self, backup_path):
+ calls.append(backup_path)
+ if len(calls) == 1:
+ raise error
+ return real(self, backup_path)
+
+ return patch.object(BackupService, "_classify_backup_key", classify), calls
+
+
+ "error",
+ [
+ OSError(5, "Input/output error"),
+ sqlite3.DatabaseError("file is not a database"),
+ sqlcipher_dbapi.DatabaseError("file is not a database"),
+ ValueError("unexpected salt size"),
+ ],
+ ids=["oserror", "sqlite3", "sqlcipher3", "valueerror"],
+)
+def test_purge_continues_past_a_backup_that_raises(manager, error):
+ # Given two old-key backups, the first examined of which raises
+ username, db_path = _user(manager)
+ backups = [_backup(username, OLD), _backup(username, OLD)]
+ patcher, calls = _first_call_raises(error)
+
+ # When the purge runs under the new password
+ with patcher:
+ deleted = BackupService(
+ username=username, password=NEW
+ ).purge_backups_under_superseded_key()
+
+ # Then the failing backup is kept and the other is still classified
+ # and deleted
+ assert len(calls) == 2
+ assert deleted == 1
+ survivors = [b for b in backups if b.exists()]
+ assert survivors == [calls[0]]
+
+
+def test_purge_still_surfaces_unexpected_errors(manager):
+ # Given two old-key backups, the first examined of which hits a bug
+ username, db_path = _user(manager)
+ backups = [_backup(username, OLD), _backup(username, OLD)]
+ patcher, calls = _first_call_raises(RuntimeError("bug"))
+
+ # When the purge runs, the error is not swallowed
+ with patcher, pytest.raises(RuntimeError, match="bug"):
+ BackupService(
+ username=username, password=NEW
+ ).purge_backups_under_superseded_key()
+
+ # And nothing was deleted on the way out
+ assert all(b.exists() for b in backups)
+
+
+_SERVICE_LOGGER = "local_deep_research.database.backup.backup_service.logger"
+
+
+def _warnings(log):
+ return [str(call.args[0]) for call in log.warning.call_args_list]
+
+
+def test_purge_continues_when_a_damaged_backup_cannot_be_moved_aside(manager):
+ # Given two old-key backups, the first examined classified as damaged
+ username, db_path = _user(manager)
+ backups = [_backup(username, OLD), _backup(username, OLD)]
+ real = BackupService._classify_backup_key
+ examined = []
+
+ def classify(self, backup_path):
+ examined.append(backup_path)
+ return "damaged" if len(examined) == 1 else real(self, backup_path)
+
+ # When the move-aside fails (returns None) during the purge
+ with (
+ patch.object(BackupService, "_classify_backup_key", classify),
+ patch.object(
+ BackupService, "_move_damaged_backup_aside", return_value=None
+ ) as move_aside,
+ patch(_SERVICE_LOGGER) as log,
+ ):
+ deleted = BackupService(
+ username=username, password=NEW
+ ).purge_backups_under_superseded_key()
+
+ # Then the damaged backup is kept and reported, and the loop carries on
+ # to delete the other superseded backup
+ move_aside.assert_called_once_with(examined[0])
+ assert any("could not be moved aside" in w for w in _warnings(log))
+ assert len(examined) == 2
+ assert deleted == 1
+ assert [b for b in backups if b.exists()] == [examined[0]]
+
+
+def test_purge_continues_when_deleting_a_superseded_backup_fails(manager):
+ # Given two superseded backups, the first removal of which raises OSError
+ username, db_path = _user(manager)
+ backups = [_backup(username, OLD), _backup(username, OLD)]
+ real = BackupService._remove_backup_files
+ attempted = []
+
+ def remove(backup_path):
+ attempted.append(backup_path)
+ if len(attempted) == 1:
+ raise OSError(13, "Permission denied")
+ real(backup_path)
+
+ # When the purge runs under the new password
+ with (
+ patch.object(
+ BackupService, "_remove_backup_files", staticmethod(remove)
+ ),
+ patch(_SERVICE_LOGGER) as log,
+ ):
+ deleted = BackupService(
+ username=username, password=NEW
+ ).purge_backups_under_superseded_key()
+
+ # Then the failed deletion is reported and not counted, the backup is
+ # kept, and the loop still deletes the other one
+ assert any("Could not delete backup" in w for w in _warnings(log))
+ assert len(attempted) == 2
+ assert deleted == 1
+ assert [b for b in backups if b.exists()] == [attempted[0]]
+
+
def _rekey_live_database(db_path, old, new):
from local_deep_research.database.sqlcipher_utils import set_sqlcipher_rekey