~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Last backup attempt, stored in the encrypted user database.
Status is written through the user's already-open encrypted connection. A
caller that is itself holding the user's new password (the password-change
purge) supplies a session on a connection it opens with that password, which
is tried first: that caller holds the rekey sidecar, and the open-engine path
would wait on that same sidecar. Status never reopens a database with a
cached password on its own, and never falls back to the plaintext auth
database.
When no encrypted connection is usable, the observation is kept only in a
bounded per-process cache and reported as unpersisted. It is written at the
next opportunity: the next recorded attempt, a status read, or a daily-reuse
backup run while the user's database is open.
Two independent facts are merged per column, so workers can write in any
order:
* the latest attempt (``completed_at``, ``success``, ``error_code``):
the newest ``completed_at`` wins;
* the password-change purge state (``purge_incomplete``,
``purge_checked_at``): only a purge sets it, and the newest purge wins,
except that on a timestamp tie or step-back "incomplete" wins.
Only a purge observation that has not been written yet travels with a write.
An ordinary backup carries such an observation from its own worker's cache;
a stored purge state is never resent. So an ordinary backup never clears a
pending warning. It can make an older "incomplete" from its own worker's
cache, never written until then, override a newer clean purge from another
worker: deliberately, since a spurious warning is cleared by the next
verified purge while a spurious all-clear would hide old-password backups.
"""
from collections import OrderedDict
from contextlib import contextmanager
from datetime import UTC, datetime
from enum import Enum
import errno
import threading
import sys
from loguru import logger
from sqlalchemy import (
Boolean,
Column,
MetaData,
String,
Table,
Integer,
case,
func,
inspect,
select,
type_coerce,
)
from sqlalchemy.dialects.sqlite import insert
from sqlalchemy.schema import CreateTable
from sqlalchemy_utc import UtcDateTime
class BackupError(str, Enum):
SOURCE_MISSING = "source_missing"
INSUFFICIENT_SPACE = "insufficient_space"
STORAGE_UNAVAILABLE = "storage_unavailable"
PERMISSION_DENIED = "permission_denied"
CONFIGURATION = "configuration"
VERIFICATION = "verification"
PUBLICATION = "publication"
SCHEDULING = "scheduling"
PURGE_INCOMPLETE = "purge_incomplete"
UNKNOWN = "unknown"
_ERROR_MESSAGES = {
BackupError.SOURCE_MISSING: "The source database could not be found.",
BackupError.INSUFFICIENT_SPACE: "There is not enough free disk space for a backup.",
BackupError.STORAGE_UNAVAILABLE: "Backup storage could not be accessed.",
BackupError.PERMISSION_DENIED: "The server lacks permission to create the backup.",
BackupError.CONFIGURATION: "The backup configuration could not be used.",
BackupError.VERIFICATION: "The backup failed verification and was discarded.",
BackupError.PUBLICATION: "The backup changed during publication and was discarded.",
BackupError.SCHEDULING: "The backup could not be scheduled.",
BackupError.PURGE_INCOMPLETE: "Cleanup of backups made before the password change is incomplete. Old-password backups may remain.",
BackupError.UNKNOWN: "The backup could not be created.",
}
# Separate metadata permits best-effort status writes without depending on
# a user-schema migration succeeding first. This table is never in auth DB.
_attempts = Table(
"user_backup_attempt_status",
MetaData(),
Column("username", String(80), primary_key=True),
Column("completed_at", UtcDateTime, nullable=False),
Column("success", Boolean, nullable=False),
Column("error_code", String(32)),
Column("purge_incomplete", Boolean, nullable=False, default=False),
# When the purge state above was last determined by a purge; NULL if no
# purge has run. Orders purge observations independently of attempts.
Column("purge_checked_at", UtcDateTime),
)
# If storage is full/unavailable, still expose the latest local observation.
# Namespace by encrypted DB root so a data-root change cannot expose old state.
_recent: OrderedDict[tuple[str, str], dict] = OrderedDict()
_recent_lock = threading.Lock()
_RECENT_LIMIT = 1024
def error_from_exception(exc: Exception) -> BackupError:
"""Classify a failure without retaining its possibly sensitive text."""
if isinstance(exc, OSError):
if exc.errno == errno.ENOSPC:
return BackupError.INSUFFICIENT_SPACE
if isinstance(exc, PermissionError):
return BackupError.PERMISSION_DENIED
return BackupError.STORAGE_UNAVAILABLE
if isinstance(exc, ValueError):
return BackupError.CONFIGURATION
return BackupError.UNKNOWN
def _purge_wins(new: dict, old: dict) -> bool:
"""Whether ``new``'s purge state replaces ``old``'s (as in _save_attempt).
A strictly newer purge wins. On a timestamp tie, or an older timestamp
from a clock step-back or a delayed worker, "incomplete" wins over
"clean": a spurious warning is fixed by repeating the purge, while a
spurious all-clear hides old-password backups.
"""
new_at, old_at = new.get("purge_checked_at"), old.get("purge_checked_at")
if new_at is None:
return False
if old_at is None or new_at > old_at:
return True
new_pending = bool(new.get("purge_incomplete"))
if new_pending != bool(old.get("purge_incomplete")):
return new_pending
return new_at == old_at # The same observation: take the newer copy.
def _remember(key: tuple[str, str], attempt: dict) -> None:
"""Merge ``attempt`` into the local observation, fact by fact."""
with _recent_lock:
previous = _recent.get(key)
if previous is not None:
latest = (
attempt
if attempt["completed_at"] >= previous["completed_at"]
else previous
)
purge = attempt if _purge_wins(attempt, previous) else previous
# A delayed write acknowledgement carries a snapshot of stored
# state, not a fresh purge observation. It cannot supersede a
# newer purge still waiting to be written by another thread.
if (
attempt.get("purge_persisted", True)
and not previous.get("purge_persisted", True)
and (
attempt.get("purge_checked_at") is None
or previous["purge_checked_at"]
> attempt["purge_checked_at"]
)
):
purge = previous
attempt = {
"completed_at": latest["completed_at"],
"success": latest["success"],
"error_code": latest["error_code"],
"persisted": latest.get("persisted", False),
"purge_incomplete": bool(purge.get("purge_incomplete")),
"purge_checked_at": purge.get("purge_checked_at"),
"purge_persisted": purge.get("purge_persisted", True),
}
_recent[key] = attempt
_recent.move_to_end(key)
while len(_recent) > _RECENT_LIMIT:
# The oldest entry of the lowest retention rank; min() keeps the
# first (oldest) of equal ranks.
del _recent[min(_recent, key=lambda k: _retention_rank(_recent[k]))]
def _retention_rank(entry: dict) -> int:
"""How strongly the bounded cache keeps ``entry``; higher is kept longer.
An observation already written can be read back from the database, so
it goes first. An unwritten purge state is kept longest: losing it would
drop a warning that old-password backups may remain. Within a rank the
oldest entry goes first, so when every entry holds an unwritten purge
the newest ones are kept.
"""
if not entry.get("purge_persisted", True):
return 2
return 1 if _is_pending(entry) else 0
def _status_key(username: str) -> tuple[str, str]:
from ...config.paths import get_data_directory
# Namespace resolution must not create a directory just to remember an
# observation in memory (including before the first encrypted DB opens).
return str(get_data_directory() / "encrypted_databases"), username
@contextmanager
def _encrypted_status_session(username: str):
# A status observation cannot initialize a database manager. An available
# encrypted user connection necessarily belongs to the already-loaded one.
module = sys.modules.get(__name__.rsplit(".", 2)[0] + ".encrypted_db")
db_manager = getattr(module, "db_manager", None)
if db_manager is None:
raise RuntimeError("Encrypted backup status is unavailable")
if not db_manager.has_encryption or not db_manager.is_user_connected(
username
):
raise RuntimeError("Encrypted backup status is unavailable")
session = db_manager.get_session(username) # noqa: raw-session
if session is None:
raise RuntimeError("Encrypted backup status is unavailable")
with session:
yield session
def _save_attempt(session, username: str, attempt: dict) -> dict:
"""Merge ``attempt`` into the stored row and return the merged row.
Each fact is taken from whichever side observed it last (see the module
docstring), so a delayed or repeated write never loses newer state.
"""
session.execute(CreateTable(_attempts, if_not_exists=True))
# Send the purge state only while it is unwritten. A cached copy of a
# stored state is stale once another worker has stored a newer one, and
# the step-back rule below would let a stale "incomplete" win.
purge_unwritten = not attempt.get("purge_persisted", True)
statement = insert(_attempts).values(
username=username,
completed_at=attempt["completed_at"],
success=attempt["success"],
error_code=attempt["error_code"],
purge_incomplete=purge_unwritten and bool(attempt["purge_incomplete"]),
purge_checked_at=(
attempt["purge_checked_at"] if purge_unwritten else None
),
)
new = statement.excluded
newer_attempt = new.completed_at >= _attempts.c.completed_at
# Mirrors _purge_wins: newer wins; on a tie or step-back, True wins.
newer_purge = new.purge_checked_at.is_not(None) & (
_attempts.c.purge_checked_at.is_(None)
| (new.purge_checked_at > _attempts.c.purge_checked_at)
| (new.purge_incomplete & ~_attempts.c.purge_incomplete)
)
# SQLite evaluates every SET expression against the original row.
update = {
name: case((newer_attempt, new[name]), else_=_attempts.c[name])
for name in ("completed_at", "success", "error_code")
}
update.update(
{
name: case((newer_purge, new[name]), else_=_attempts.c[name])
for name in ("purge_incomplete", "purge_checked_at")
}
)
session.execute(
statement.on_conflict_do_update(
index_elements=["username"], set_=update
)
)
try:
row = dict(
session.execute(
select(_attempts).where(_attempts.c.username == username)
)
.mappings()
.one()
)
except (ValueError, TypeError, OverflowError) as exc:
row = _replace_unreadable_row(session, username, attempt, exc)
session.commit()
return {**row, "persisted": True, "purge_persisted": True}
def _replace_unreadable_row(session, username, attempt, exc) -> dict:
"""Repair a stored row whose values cannot be parsed, after the upsert.
One corrupt value (say a malformed ``completed_at``) would otherwise make
every later write fail on the read-back, and its text comparison could
keep the merge from ever replacing it. The upsert above has already
merged this write into the row, so each column that still reads cleanly
keeps its merged value. Only an unreadable column is replaced:
* the attempt columns, together, by this write's attempt;
* ``purge_incomplete`` by "incomplete", since nothing proves a clean
purge (a newer purge this write carried was already written by the
upsert, which leaves a readable flag);
* ``purge_checked_at`` by NULL, so any later purge decides. The merge
could not order against an unreadable timestamp, so a purge state this
write carries then replaces both purge columns instead.
"""
logger.warning(
"Repairing an unreadable backup status row ({})", type(exc).__name__
)
owner = _attempts.c.username == username
carried = not attempt.get("purge_persisted", True)
def read(column):
try:
return True, session.execute(
select(column).where(owner)
).scalar_one()
except (ValueError, TypeError, OverflowError):
return False, None
row = {}
attempt_columns = ("completed_at", "success", "error_code")
values = [read(_attempts.c[name]) for name in attempt_columns]
if all(ok for ok, _ in values):
row.update(zip(attempt_columns, (value for _, value in values)))
else:
row.update(
completed_at=attempt["completed_at"],
success=bool(attempt["success"]),
error_code=attempt["error_code"],
)
checked_ok, checked = read(_attempts.c.purge_checked_at)
if carried and not checked_ok:
row["purge_incomplete"] = bool(attempt["purge_incomplete"])
row["purge_checked_at"] = attempt["purge_checked_at"]
else:
# Boolean results are not validated on read, so check the storage:
# only an integer 0 or 1 is a readable flag.
kind, flag = session.execute(
select(
func.typeof(_attempts.c.purge_incomplete),
type_coerce(_attempts.c.purge_incomplete, Integer),
).where(owner)
).one()
# A purge this write carries that is newer than the stored one was
# already written as an integer by the upsert, so an unreadable flag
# here is a stored state nothing has superseded: assume incomplete.
row["purge_incomplete"] = (
bool(flag) if kind == "integer" and flag in (0, 1) else True
)
row["purge_checked_at"] = checked if checked_ok else None
session.execute(_attempts.update().where(owner).values(**row))
return {"username": username, **row}
def _is_pending(entry: dict | None) -> bool:
"""Whether a local observation holds anything not yet written."""
return entry is not None and not (
entry.get("persisted") and entry.get("purge_persisted", True)
)
def _persist(username: str, key, attempt: dict, direct_session=None) -> None:
"""Write ``attempt``, or raise if no encrypted session is usable.
``direct_session`` is tried first, then the open engine. The open engine
is avoided under the rekey sidecar only while ``direct_session`` works:
if it fails while the caller holds the sidecar, the fallback can wait on
that sidecar for up to its wait bound (it does not deadlock). A write
that fails leaves the observation pending in the local cache.
"""
saved = None
if direct_session is not None:
try:
with direct_session() as session:
saved = _save_attempt(session, username, attempt)
except Exception as exc:
logger.debug(
"Direct backup status session unusable ({})",
type(exc).__name__,
)
if saved is None:
with _encrypted_status_session(username) as session:
saved = _save_attempt(session, username, attempt)
_remember(key, saved)
def record_backup_result(
username: str,
success: bool,
error_code: BackupError | None = None,
*,
purge_incomplete: bool | None = None,
direct_session=None,
) -> None:
"""Best-effort reporting must never change a backup's actual outcome.
``purge_incomplete`` is given only by a password-change purge: True when
old-password files may remain, False when the purge verified none did.
``direct_session``, a zero-argument callable returning a session context
manager on the user's encrypted database, is tried before the open
engine (see the module docstring).
"""
if not username:
return
try:
key = _status_key(username)
code = (
None if success else BackupError(error_code or BackupError.UNKNOWN)
)
now = datetime.now(UTC)
purge, purge_checked_at, purge_persisted = False, None, True
if purge_incomplete is not None:
purge, purge_checked_at, purge_persisted = (
bool(purge_incomplete),
now,
False,
)
else:
# Carry a purge state not yet written, with its own timestamp,
# so this write persists it. A state already stored is not
# resent: the database may have a newer one from another worker.
with _recent_lock:
previous = _recent.get(key, {})
if not previous.get("purge_persisted", True):
purge = bool(previous.get("purge_incomplete"))
purge_checked_at = previous.get("purge_checked_at")
purge_persisted = False
attempt = {
"completed_at": now,
"success": bool(success),
"error_code": code.value if code else None,
"purge_incomplete": purge,
"purge_checked_at": purge_checked_at,
"persisted": False,
"purge_persisted": purge_persisted,
}
_remember(key, attempt)
_persist(username, key, attempt, direct_session)
except Exception as exc:
logger.warning(
"Could not save encrypted backup status ({})", type(exc).__name__
)
def persist_pending_backup_status(username: str) -> None:
"""Write an unpersisted local observation if the user DB is now open.
Called where a backup run ends without recording a new attempt (the
daily-reuse early return), so an observation made while the database
was closed, such as an incomplete password-change purge, does not stay
process-local and vanish on restart.
"""
if not username:
return
try:
key = _status_key(username)
with _recent_lock:
local = _recent.get(key)
if not _is_pending(local):
return
_persist(username, key, local)
except Exception as exc:
logger.debug(
"Pending backup status not yet saved ({})", type(exc).__name__
)
def get_backup_status(username: str) -> dict:
"""Read this account's attempt; never substitute a plaintext store."""
available = True
attempt = None
local = None
try:
key = _status_key(username)
with _recent_lock:
local = _recent.get(key)
with _encrypted_status_session(username) as session:
if _is_pending(local):
# Persist an observation made before the encrypted DB
# opened; the merged row comes back (and an unreadable one
# is replaced).
attempt = _save_attempt(session, username, local)
_remember(key, attempt)
elif inspect(session.connection()).has_table(_attempts.name):
row = (
session.execute(
select(_attempts).where(
_attempts.c.username == username
)
)
.mappings()
.first()
)
if row is not None:
attempt = {**row, "persisted": True}
except Exception as exc:
available = False
attempt = local
logger.warning(
"Could not read encrypted backup status ({})", type(exc).__name__
)
try:
attempt = _report(attempt)
except Exception as exc:
# A malformed stored row must not break the settings panel.
available, attempt = False, None
logger.warning(
"Could not format backup status ({})", type(exc).__name__
)
return {"status_available": available, "last_attempt": attempt}
def _report(attempt: dict | None) -> dict | None:
"""Turn a stored or local observation into the public report."""
if attempt is not None:
pending = bool(attempt.get("purge_incomplete"))
success = bool(attempt["success"]) and not pending
try:
code = (
BackupError.PURGE_INCOMPLETE
if pending and attempt["success"]
else BackupError(attempt["error_code"] or BackupError.UNKNOWN)
)
except ValueError:
code = BackupError.UNKNOWN
error = None if success else _ERROR_MESSAGES[code]
if pending and code != BackupError.PURGE_INCOMPLETE:
error = (
(error or "")
+ " "
+ _ERROR_MESSAGES[BackupError.PURGE_INCOMPLETE]
)
attempt = {
"completed_at": attempt["completed_at"].isoformat(),
"success": success,
"error_code": None if success else code.value,
"error": error,
# Explicit, because error_code names the attempt's own failure
# when a failed backup and an incomplete purge coincide.
"purge_incomplete": pending,
}
return attempt