~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(backup): keep superseded-key purge going past a backup that raises (#7748)
Imaduddin Bhaisaheb82e867b5e8b6
diff --git a/changelog.d/7678.bugfix.md b/changelog.d/7678.bugfix.mdnew file mode 100644index 0000000000..d747ec44c7--- /dev/null+++ b/changelog.d/7678.bugfix.md@@ -0,0 +1 @@+The superseded-key backup purge now guarantees that a backup that cannot be read or classified is kept and never stops it from examining the remaining old-key backups.diff --git a/src/local_deep_research/database/backup/backup_service.py b/src/local_deep_research/database/backup/backup_service.pyindex 61e0467a4d..6b2e2ef5fc 100644--- a/src/local_deep_research/database/backup/backup_service.py+++ b/src/local_deep_research/database/backup/backup_service.py@@ -8,6 +8,7 @@ import hmacimport osimport reimport shutil+import sqlite3import tempfileimport threadingimport time@@ -82,6 +83,23 @@ _QUARANTINED_BACKUP_NAME_RE = re.compile()+# Errors one corrupt or unreadable backup can raise while it is classified,+# moved aside or deleted. sqlcipher3's errors do not subclass sqlite3.Error,+# so its DB-API base class is listed too.+try:+ from sqlcipher3 import dbapi2 as _sqlcipher_dbapi++ _SQLCIPHER_ERRORS: tuple = (_sqlcipher_dbapi.Error,)+except ImportError: # pragma: no cover - SQLCipher optional+ _SQLCIPHER_ERRORS = ()+_PURGE_ITEM_ERRORS = (+ OSError,+ sqlite3.Error,+ ValueError,+ *_SQLCIPHER_ERRORS,+)++def _validate_backup_path(path: Path) -> str:"""Normalize host separators and validate the path used in ATTACH.@@ -1281,6 +1299,11 @@ class BackupService:salt its database needs is left to the regular verification,which already rejects it.+ A backup that raises while it is examined (an I/O error, a+ SQLite/SQLCipher error, an unreadable salt) is kept and logged;+ the purge carries on with the remaining backups. Errors outside+ ``_PURGE_ITEM_ERRORS`` are bugs and propagate.+Application callers must hold this database's rekey sidecar lockand establish that this password still belongs to its currentstable epoch, through the entire purge. The per-user backup lock@@ -1294,45 +1317,64 @@ class BackupService:_get_backup_verifications(self.username).clear()source_has_salt = has_per_database_salt(self.db_path)for backup_path in self._managed_backup_paths():- backup_salt = get_salt_file_path(backup_path)- if (- not backup_path.is_file()- or backup_path.is_symlink()- or backup_salt.is_symlink()- or (source_has_salt and not backup_salt.exists())- ):- continue- verdict = self._classify_backup_key(backup_path)- if verdict == "damaged":- if _QUARANTINED_BACKUP_NAME_RE.fullmatch(backup_path.name):- continue- moved = self._move_damaged_backup_aside(backup_path)- if moved is None:- logger.warning(- f"Backup {backup_path.name} is under the current "- "key but damaged, and could not be moved aside"- )- else:- logger.warning(- f"Backup {backup_path.name} is under the current "- f"key but its first page is damaged; moved aside "- f"as {moved.name}"- )- continue- if verdict != "superseded":- continue+ # One unreadable or corrupt backup must not shield the rest+ # from the purge: it is kept (fail-safe) and the loop moves+ # on. Only the errors a bad backup can raise are caught;+ # anything else is a bug and still propagates.try:- self._remove_backup_files(backup_path)- deleted += 1- logger.info(- f"Purged backup under superseded key: {backup_path.name}"- )- except OSError:+ if self._purge_one_superseded_backup(+ backup_path, source_has_salt+ ):+ deleted += 1+ except _PURGE_ITEM_ERRORS as error:logger.warning(- f"Could not delete backup {backup_path.name}"+ f"Skipping backup {backup_path.name} in the "+ f"superseded-key purge: {type(error).__name__}")return deleted+ def _purge_one_superseded_backup(+ self, backup_path: Path, source_has_salt: bool+ ) -> bool:+ """Classify one backup and delete it or move it aside.++ Returns True only when the backup was deleted as superseded.+ """+ backup_salt = get_salt_file_path(backup_path)+ if (+ not backup_path.is_file()+ or backup_path.is_symlink()+ or backup_salt.is_symlink()+ or (source_has_salt and not backup_salt.exists())+ ):+ return False+ verdict = self._classify_backup_key(backup_path)+ if verdict == "damaged":+ if _QUARANTINED_BACKUP_NAME_RE.fullmatch(backup_path.name):+ return False+ moved = self._move_damaged_backup_aside(backup_path)+ if moved is None:+ logger.warning(+ f"Backup {backup_path.name} is under the current "+ "key but damaged, and could not be moved aside"+ )+ else:+ logger.warning(+ f"Backup {backup_path.name} is under the current "+ f"key but its first page is damaged; moved aside "+ f"as {moved.name}"+ )+ return False+ if verdict != "superseded":+ return False+ try:+ self._remove_backup_files(backup_path)+ except OSError:+ logger.warning(f"Could not delete backup {backup_path.name}")+ return False+ logger.info(f"Purged backup under superseded key: {backup_path.name}")+ return True+def get_latest_backup(self) -> Optional[Path]:"""Get the path to the most recent backup.diff --git a/tests/database/backup/test_superseded_key_purge.py b/tests/database/backup/test_superseded_key_purge.pyindex e076278285..1935796653 100644--- a/tests/database/backup/test_superseded_key_purge.py+++ b/tests/database/backup/test_superseded_key_purge.py@@ -10,6 +10,7 @@ SQLCipher throughout.import multiprocessingimport os+import sqlite3import statimport timeimport uuid@@ -39,6 +40,8 @@ from local_deep_research.database.sqlcipher_utils import (set_sqlcipher_key,)+sqlcipher_dbapi = pytest.importorskip("sqlcipher3.dbapi2")+OLD = "OldPurgeHorse1!" # noqa: S105NEW = "NewPurgeHorse2!" # noqa: S105LATEST = "LatestPurgeHorse3!" # noqa: S105@@ -323,6 +326,137 @@ def test_purge_moves_a_damaged_current_key_backup_aside(manager):assert moved_db[0].exists()+def _first_call_raises(error):+ """Patch ``_classify_backup_key`` so its first call raises ``error``."""+ real = BackupService._classify_backup_key+ calls = []++ def classify(self, backup_path):+ calls.append(backup_path)+ if len(calls) == 1:+ raise error+ return real(self, backup_path)++ return patch.object(BackupService, "_classify_backup_key", classify), calls+++ "error",+ [+ OSError(5, "Input/output error"),+ sqlite3.DatabaseError("file is not a database"),+ sqlcipher_dbapi.DatabaseError("file is not a database"),+ ValueError("unexpected salt size"),+ ],+ ids=["oserror", "sqlite3", "sqlcipher3", "valueerror"],+)+def test_purge_continues_past_a_backup_that_raises(manager, error):+ # Given two old-key backups, the first examined of which raises+ username, db_path = _user(manager)+ backups = [_backup(username, OLD), _backup(username, OLD)]+ patcher, calls = _first_call_raises(error)++ # When the purge runs under the new password+ with patcher:+ deleted = BackupService(+ username=username, password=NEW+ ).purge_backups_under_superseded_key()++ # Then the failing backup is kept and the other is still classified+ # and deleted+ assert len(calls) == 2+ assert deleted == 1+ survivors = [b for b in backups if b.exists()]+ assert survivors == [calls[0]]+++def test_purge_still_surfaces_unexpected_errors(manager):+ # Given two old-key backups, the first examined of which hits a bug+ username, db_path = _user(manager)+ backups = [_backup(username, OLD), _backup(username, OLD)]+ patcher, calls = _first_call_raises(RuntimeError("bug"))++ # When the purge runs, the error is not swallowed+ with patcher, pytest.raises(RuntimeError, match="bug"):+ BackupService(+ username=username, password=NEW+ ).purge_backups_under_superseded_key()++ # And nothing was deleted on the way out+ assert all(b.exists() for b in backups)+++_SERVICE_LOGGER = "local_deep_research.database.backup.backup_service.logger"+++def _warnings(log):+ return [str(call.args[0]) for call in log.warning.call_args_list]+++def test_purge_continues_when_a_damaged_backup_cannot_be_moved_aside(manager):+ # Given two old-key backups, the first examined classified as damaged+ username, db_path = _user(manager)+ backups = [_backup(username, OLD), _backup(username, OLD)]+ real = BackupService._classify_backup_key+ examined = []++ def classify(self, backup_path):+ examined.append(backup_path)+ return "damaged" if len(examined) == 1 else real(self, backup_path)++ # When the move-aside fails (returns None) during the purge+ with (+ patch.object(BackupService, "_classify_backup_key", classify),+ patch.object(+ BackupService, "_move_damaged_backup_aside", return_value=None+ ) as move_aside,+ patch(_SERVICE_LOGGER) as log,+ ):+ deleted = BackupService(+ username=username, password=NEW+ ).purge_backups_under_superseded_key()++ # Then the damaged backup is kept and reported, and the loop carries on+ # to delete the other superseded backup+ move_aside.assert_called_once_with(examined[0])+ assert any("could not be moved aside" in w for w in _warnings(log))+ assert len(examined) == 2+ assert deleted == 1+ assert [b for b in backups if b.exists()] == [examined[0]]+++def test_purge_continues_when_deleting_a_superseded_backup_fails(manager):+ # Given two superseded backups, the first removal of which raises OSError+ username, db_path = _user(manager)+ backups = [_backup(username, OLD), _backup(username, OLD)]+ real = BackupService._remove_backup_files+ attempted = []++ def remove(backup_path):+ attempted.append(backup_path)+ if len(attempted) == 1:+ raise OSError(13, "Permission denied")+ real(backup_path)++ # When the purge runs under the new password+ with (+ patch.object(+ BackupService, "_remove_backup_files", staticmethod(remove)+ ),+ patch(_SERVICE_LOGGER) as log,+ ):+ deleted = BackupService(+ username=username, password=NEW+ ).purge_backups_under_superseded_key()++ # Then the failed deletion is reported and not counted, the backup is+ # kept, and the loop still deletes the other one+ assert any("Could not delete backup" in w for w in _warnings(log))+ assert len(attempted) == 2+ assert deleted == 1+ assert [b for b in backups if b.exists()] == [attempted[0]]++def _rekey_live_database(db_path, old, new):from local_deep_research.database.sqlcipher_utils import set_sqlcipher_rekey