~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Cross-process serialization for SQLCipher database rekeys."""
from __future__ import annotations
import errno
import os
import stat
import tempfile
import threading
import time
import weakref
from collections.abc import Callable, Generator
from contextlib import contextmanager
from dataclasses import dataclass
from enum import StrEnum, unique
from pathlib import Path
from typing import TypeVar
from uuid import UUID, uuid4
class RekeyOperationalError(Exception):
"""Base for explicit operational failures during password rekey."""
@unique
class RekeySidecarOperation(StrEnum):
"""Safe lifecycle stages for sidecar I/O diagnostics."""
OPEN = "open"
ACQUIRE = "acquire"
UNLOCK = "unlock"
CLOSE = "close"
# Writing an epoch token (``.rekey.epoch`` / ``.rekey.pending``) while
# the sidecar is held: mkstemp, write+fsync, replace, directory sync.
PUBLISH = "publish"
# Errnos this module raises itself when it refuses a planted sidecar.
_SIDECAR_REFUSAL_DETAIL: dict[int | None, str] = {
errno.ELOOP: "; refusing a symlinked sidecar",
errno.EINVAL: "; refusing a non-regular sidecar",
# st_nlink == 0 (an already-unlinked inode), or a hard link to a
# database file or any other file in the sidecar's directory
errno.EMLINK: "; refusing an unlinked sidecar or one hard-linked within its directory",
}
class RekeySidecarIOError(RekeyOperationalError):
"""Normalized sidecar I/O failure without raw operating-system text."""
db_path: Path
operation: RekeySidecarOperation
errno: int | None
def __init__(
self,
db_path: Path,
operation: RekeySidecarOperation,
error_number: int | None,
) -> None:
self.db_path = db_path
self.operation = operation
self.errno = error_number
errno_text = "unknown" if error_number is None else str(error_number)
detail = _SIDECAR_REFUSAL_DETAIL.get(error_number, "")
super().__init__(
f"rekey sidecar {operation.value} failed "
f"(errno={errno_text}{detail})"
)
@unique
class RekeyStage(StrEnum):
"""Closed stages used to classify password-rekey outcomes."""
PREPARATION = "preparation"
PENDING_PUBLICATION = "pending-publication"
PHYSICAL_REKEY = "physical-rekey"
CANDIDATE_VERIFICATION = "candidate-verification"
COMMIT_PUBLICATION = "commit-publication"
class RekeyUnavailableError(RekeyOperationalError):
"""Raised when rekey cannot proceed before its outcome becomes ambiguous."""
db_path: Path
stage: RekeyStage
def __init__(self, db_path: Path, stage: RekeyStage) -> None:
self.db_path = db_path
self.stage = stage
super().__init__(
f"rekey unavailable during {stage.value} for {db_path.name}"
)
class RekeyIndeterminateError(RekeyOperationalError):
"""Raised when durable state cannot prove which password remains active."""
db_path: Path
stage: RekeyStage
state: RekeyEpochState | None
def __init__(
self,
db_path: Path,
stage: RekeyStage,
state: RekeyEpochState | None,
) -> None:
self.db_path = db_path
self.stage = stage
self.state = state
super().__init__(
f"rekey outcome indeterminate during {stage.value} for {db_path.name}"
)
class RekeyLockContentionError(RekeyOperationalError):
"""Raised when another password rekey already owns the database lock."""
lock_path: Path
def __init__(self, lock_path: Path) -> None:
self.lock_path = lock_path
super().__init__(f"rekey already in progress for {lock_path.name}")
class DatabaseKeyCheckUnavailableError(RekeyOperationalError):
"""The password could not be checked against the database right now.
Raised when opening the database to verify a password fails for an
operational reason (SQLite reports the database locked or busy past the
busy timeout, an I/O error, a file it cannot open), as opposed to the
key being wrong ("file is not a database"). It says nothing about the
password, so a login answers 503 without a lockout strike and a password
change answers with its retryable failure, never "password incorrect".
Only the SQLite exception type is kept, not its message.
"""
db_path: Path
reason: str
def __init__(self, db_path: Path, reason: str) -> None:
self.db_path = db_path
self.reason = reason
super().__init__(
f"password check unavailable for {db_path.name} ({reason})"
)
class RekeyRecoveryRequiredError(RekeyOperationalError):
"""Raised when durable rekey metadata requires verified recovery."""
db_path: Path
def __init__(self, db_path: Path) -> None:
self.db_path = db_path
super().__init__(f"rekey recovery required for {db_path.name}")
class RekeyEpochUnreadableError(RekeyRecoveryRequiredError):
"""An epoch file exists (or may) but the read itself failed.
Raised for every ``OSError`` other than ``FileNotFoundError`` while
reading ``.rekey.epoch`` / ``.rekey.pending`` (EMFILE, ENFILE, EAGAIN,
EIO, EACCES, ...; EINTR never gets here, Python retries it itself per
PEP 475; a PermissionError is first retried a bounded number of times
as a possible Windows sharing violation, see
``_retry_sharing_violation``), and when the reader refuses a planted
file without reading it: a symlink (ELOOP) or anything that is not a
regular file, such as a FIFO or directory (EINVAL). It is a
``RekeyRecoveryRequiredError``, so
every existing handler keeps failing closed exactly as before; the
subclass only lets a caller that can retry later (session validation)
tell "nothing could be read" apart from durable state that was read
and does not vouch for the identity. ``errno`` is kept for
diagnostics; the raw OS message is not.
"""
errno: int | None
def __init__(self, db_path: Path, error_number: int | None) -> None:
super().__init__(db_path)
self.errno = error_number
def describe_rekey_fault(error: RekeyOperationalError) -> str:
"""Scrub-safe ``type (operation=..., errno=...)`` summary of a rekey fault.
Shared by the login route's log line and ``DatabaseSessionUnavailableError``
so an operator sees the same fields for the same fault on either path.
Only the exception's type name, the closed ``RekeySidecarOperation`` value
and the numeric errno are used. Never a path, the OS error text or the
exception message, which may carry path material.
"""
fields: list[str] = []
if isinstance(error, RekeySidecarIOError):
fields.append(f"operation={error.operation.value}")
if isinstance(error, (RekeySidecarIOError, RekeyEpochUnreadableError)):
error_number = error.errno
fields.append(
f"errno={'unknown' if error_number is None else error_number}"
)
name = type(error).__name__
return f"{name} ({', '.join(fields)})" if fields else name
@dataclass(frozen=True, slots=True)
class RekeyEpochState:
"""Durable committed and in-flight rekey generations."""
committed_epoch: str | None
pending_epoch: str | None
@dataclass(frozen=True, slots=True)
class _SidecarLockRequest:
db_path: Path
lock_path: Path
blocking: bool
gate_wait: float | None = None
contention_retry: float = 0.0
# time.monotonic() value bounding the gate wait and the flock retries
# together; see acquire_rekey_lock's ``timeout``.
deadline: float | None = None
# Poll interval while a non-blocking caller rides out a brief foreign hold.
_CONTENTION_RETRY_INTERVAL = 0.01
# In-process gate per sidecar file. flock semantics make two fds in one
# process contend like unrelated processes, so a heavyweight sidecar user
# (change_password) cannot tell "an in-process holder has the lock" from
# "a cross-process rekey holds it" and either fails fast on both or blocks
# on both. In-process holders are not all brief: cache probes hold it for
# milliseconds, but a cold open holds it across migrations and engine
# build, and a disposal across its WAL checkpoint (seconds on a large
# database). Every in-process flock holder therefore takes this gate
# first: heavyweight users wait on the gate (bounded) for local holders,
# then take the flock non-blocking (with a short retry for other
# processes' probes) so genuinely foreign holders still fail with the
# typed contention error. Request-path callers bound the gate wait and
# the flock retries together, by one deadline (``timeout``).
# Waiters and holders keep a strong local reference through gate release.
_sidecar_gates: weakref.WeakValueDictionary[Path, threading.Lock] = (
weakref.WeakValueDictionary()
)
_sidecar_gates_guard = threading.Lock()
def _get_sidecar_gate(lock_path: Path) -> threading.Lock:
with _sidecar_gates_guard:
lock = _sidecar_gates.get(lock_path)
if lock is None:
lock = threading.Lock()
_sidecar_gates[lock_path] = lock
return lock
def _acquire_posix_lock(
fd: int,
lock_path: Path,
*,
blocking: bool = False,
) -> None:
import fcntl
operation = fcntl.LOCK_EX
if not blocking:
operation |= fcntl.LOCK_NB
try:
fcntl.flock(fd, operation)
except OSError as error:
if error.errno in {errno.EACCES, errno.EAGAIN}:
contention_error = RekeyLockContentionError(lock_path)
else:
raise
else:
return
raise contention_error
def _release_posix_lock(fd: int) -> None:
import fcntl
fcntl.flock(fd, fcntl.LOCK_UN)
def _acquire_windows_lock(
fd: int,
lock_path: Path,
*,
blocking: bool = False,
) -> None:
import msvcrt
if os.fstat(fd).st_size == 0:
_ = os.lseek(fd, 0, os.SEEK_SET)
_ = os.write(fd, b"\0")
_ = os.lseek(fd, 0, os.SEEK_SET)
try:
lock_mode = msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK
msvcrt.locking(fd, lock_mode, 1)
except OSError as error:
if error.errno in {
errno.EACCES,
errno.EAGAIN,
errno.EDEADLK,
} or getattr(error, "winerror", None) in {33, 36}:
contention_error = RekeyLockContentionError(lock_path)
else:
raise
else:
return
raise contention_error
def _release_windows_lock(fd: int) -> None:
import msvcrt
_ = os.lseek(fd, 0, os.SEEK_SET)
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
def _sidecar_io_error(
lock_request: _SidecarLockRequest,
operation: RekeySidecarOperation,
error: OSError,
) -> RekeySidecarIOError:
return RekeySidecarIOError(lock_request.db_path, operation, error.errno)
def _open_sidecar(lock_request: _SidecarLockRequest) -> int:
# O_NOFOLLOW: a symlink planted at the sidecar name must not redirect the
# open (or the permission tightening below) onto an unrelated file the
# process can write; it fails with ELOOP and surfaces as a typed OPEN
# error. The mode is tightened through the fd for the same reason.
flags = os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0)
# Checked BEFORE the open: closing any descriptor to the database (or
# its WAL/shm) releases every POSIX lock this process holds on it behind
# SQLite's back, so a sidecar hard-linked to one of them must never be
# opened at all, not even to refuse it.
try:
planted = os.lstat(lock_request.lock_path)
except OSError:
planted = None
if planted is not None and _is_database_file(lock_request, planted):
raise RekeySidecarIOError(
lock_request.db_path, RekeySidecarOperation.OPEN, errno.EMLINK
)
try:
fd = os.open(lock_request.lock_path, flags, 0o600)
except OSError as error:
open_error = _sidecar_io_error(
lock_request, RekeySidecarOperation.OPEN, error
)
else:
# O_NOFOLLOW does not catch a HARD link planted at the sidecar name:
# the open lands on the linked inode and fchmod would tighten that
# unrelated file. A hard-linked sidecar is still locked (an advisory
# lock changes nothing in the file, and refusing it locked the user
# out whenever a backup or snapshot tool hard-linked the data
# directory), but its mode is left alone.
refusal = _refuse_foreign_sidecar(lock_request, fd)
if refusal is not None:
_close_refused_sidecar(fd)
raise refusal
try:
hard_linked = os.fstat(fd).st_nlink > 1
except OSError:
hard_linked = True
fchmod = None if hard_linked else getattr(os, "fchmod", None)
if fchmod is not None:
try:
fchmod(fd, 0o600)
except OSError:
return fd
return fd
raise open_error
def _is_database_file(
lock_request: _SidecarLockRequest, status: os.stat_result
) -> bool:
"""True when ``status`` may be the inode of a database file.
A lock file whose inode is also a database, WAL or shm file (this
user's or ANOTHER user's in the same directory) must never be opened:
closing the lock's descriptor releases every POSIX lock this process
holds on that inode behind SQLite's back. Only an inode with more than
one link can be one. This user's own files are stat'ed directly; every
other entry of the sidecar's directory is then compared without
following symlinks, so any other name for the inode in that directory
refuses it. Links only outside the directory (a ``cp -al`` snapshot of
the data directory) stay accepted. A directory that cannot be listed
refuses too: nothing then shows the inode is not a database.
"""
if status.st_nlink < 2:
return False
identity = (status.st_dev, status.st_ino)
db_path = lock_request.db_path
for path in (
db_path,
db_path.with_name(f"{db_path.name}-wal"),
db_path.with_name(f"{db_path.name}-shm"),
):
try:
other = os.stat(path)
except OSError:
continue
if (other.st_dev, other.st_ino) == identity:
return True
lock_name = lock_request.lock_path.name
try:
# ``lock_request`` is a ``_SidecarLockRequest``, not the HTTP
# request; ``lock_path.parent`` is the service-generated directory
# of this user's database, so nothing here is caller-supplied.
with os.scandir(lock_request.lock_path.parent) as entries:
for entry in entries:
if entry.name == lock_name:
continue
try:
other = entry.stat(follow_symlinks=False)
except OSError:
continue
if (other.st_dev, other.st_ino) == identity:
return True
except OSError:
return True
return False
def _refuse_foreign_sidecar(
lock_request: _SidecarLockRequest, fd: int
) -> RekeySidecarIOError | None:
try:
status = os.fstat(fd)
except OSError as error:
return _sidecar_io_error(
lock_request, RekeySidecarOperation.OPEN, error
)
if not stat.S_ISREG(status.st_mode):
return RekeySidecarIOError(
lock_request.db_path, RekeySidecarOperation.OPEN, errno.EINVAL
)
if _is_database_file(lock_request, status):
# Planted between the check before the open and the open itself:
# refused here as well, although the close that follows may
# already have dropped this process's locks on the database.
return RekeySidecarIOError(
lock_request.db_path, RekeySidecarOperation.OPEN, errno.EMLINK
)
if status.st_nlink == 0:
# Already unlinked: a lock on it would exclude nobody who opens the
# name afterwards.
return RekeySidecarIOError(
lock_request.db_path, RekeySidecarOperation.OPEN, errno.EMLINK
)
return None
def _close_refused_sidecar(fd: int) -> None:
try:
os.close(fd)
except OSError:
return
@contextmanager
def _acquire_sidecar_lock(
lock_request: _SidecarLockRequest,
) -> Generator[None, None, None]:
"""Hold one sidecar lock while preserving protected-body failures."""
gate = _get_sidecar_gate(lock_request.lock_path)
if lock_request.deadline is not None:
gate_timeout: float | None = max(
0.0, lock_request.deadline - time.monotonic()
)
elif lock_request.gate_wait is not None:
gate_timeout = lock_request.gate_wait
elif lock_request.blocking:
gate_timeout = None
else:
gate_timeout = 0.0
if gate_timeout is None:
acquired = gate.acquire()
else:
acquired = gate.acquire(timeout=gate_timeout)
if not acquired:
raise RekeyLockContentionError(lock_request.lock_path)
try:
yield from _acquire_sidecar_lock_locked(lock_request)
finally:
gate.release()
def _acquire_os_lock_with_retry(
fd: int, lock_request: _SidecarLockRequest
) -> None:
"""Take the OS lock, retrying a non-blocking attempt for a short window.
Every cache probe in any process (``is_user_connected``/``get_session``)
takes this flock for a few milliseconds. A single non-blocking attempt
therefore reported spurious contention whenever it collided with one of
those probes; ``contention_retry`` absorbs such brief holds while a
genuine long-lived owner (another process's rekey) still fails fast.
A lock request with a ``deadline`` retries until that deadline instead.
"""
if lock_request.deadline is not None:
deadline = lock_request.deadline
else:
deadline = time.monotonic() + lock_request.contention_retry
while True:
try:
if os.name == "nt":
_acquire_windows_lock(
fd,
lock_request.lock_path,
blocking=lock_request.blocking,
)
else:
_acquire_posix_lock(
fd,
lock_request.lock_path,
blocking=lock_request.blocking,
)
except RekeyLockContentionError:
if lock_request.blocking or time.monotonic() >= deadline:
raise
time.sleep(_CONTENTION_RETRY_INTERVAL)
else:
return
def _acquire_sidecar_lock_locked(
lock_request: _SidecarLockRequest,
) -> Generator[None, None, None]:
fd = _open_sidecar(lock_request)
acquired = False
# The failure of the acquire or of the protected body, recorded as it
# passes through. A release failure is added to it as a note, and is
# raised itself only when there is none. Not sys.exception() in the
# finally below: after a clean body, this generator is resumed from
# its caller's frame, and when that caller is inside an except block
# sys.exception() returns the exception IT is handling, so a failed
# unlock or close was attached to that unrelated exception as a note
# and never raised.
active_error: BaseException | None = None
try:
acquire_error: RekeySidecarIOError | None = None
try:
_acquire_os_lock_with_retry(fd, lock_request)
except OSError as error:
acquire_error = _sidecar_io_error(
lock_request, RekeySidecarOperation.ACQUIRE, error
)
if acquire_error is not None:
raise acquire_error # noqa: TRY301 — recorded as active_error by the except below
acquired = True
yield
except BaseException as failure:
active_error = failure
raise
finally:
cleanup_errors: list[RekeySidecarIOError] = []
if acquired:
try:
if os.name == "nt":
_release_windows_lock(fd)
else:
_release_posix_lock(fd)
except OSError as error:
cleanup_errors.append(
_sidecar_io_error(
lock_request, RekeySidecarOperation.UNLOCK, error
)
)
try:
os.close(fd)
except OSError as error:
cleanup_errors.append(
_sidecar_io_error(
lock_request, RekeySidecarOperation.CLOSE, error
)
)
if active_error is not None:
for cleanup_error in cleanup_errors:
active_error.add_note(str(cleanup_error))
elif cleanup_errors:
primary_error = cleanup_errors[0]
for cleanup_error in cleanup_errors[1:]:
primary_error.add_note(str(cleanup_error))
raise primary_error
@contextmanager
def acquire_rekey_lock(
db_path: Path,
*,
blocking: bool = False,
local_wait: float | None = None,
contention_retry: float = 0.0,
timeout: float | None = None,
) -> Generator[None, None, None]:
"""Hold the database's persistent, owner-only rekey sidecar lock.
``local_wait`` serves the heavyweight caller (change_password): wait up
to that many seconds for IN-PROCESS holders via the sidecar gate. Those
holders are not only brief cache probes: a cold open keeps the gate
through key derivation, alembic migrations and engine build, and an
engine disposal through its WAL checkpoint, so the wait is bounded
rather than assumed short. The flock is then taken non-blocking,
retried for at most ``contention_retry`` seconds to ride out other
processes' millisecond cache probes, so a genuine cross-process owner
still fails fast with ``RekeyLockContentionError``.
``timeout`` bounds the whole acquisition instead, for the callers on a
request's path (``DatabaseManager``'s cache claim, cold open and
registration): at most that many seconds in total, first on the gate
for in-process holders, then on the flock, which is retried
non-blocking until the same deadline. A holder keeps the sidecar for as
long as its work runs (a rekey or a cold open of a large database), and
a stopped process or a hung mount keeps it indefinitely, so the wait
ends there with ``RekeyLockContentionError``. It overrides ``blocking``,
``local_wait`` and ``contention_retry``.
"""
lock_request = _SidecarLockRequest(
db_path=db_path,
lock_path=db_path.with_name(f"{db_path.name}.rekey.lock"),
blocking=blocking and timeout is None,
gate_wait=local_wait,
contention_retry=contention_retry,
deadline=None if timeout is None else time.monotonic() + timeout,
)
with _acquire_sidecar_lock(lock_request):
yield
def _epoch_path(db_path: Path) -> Path:
return db_path.with_name(f"{db_path.name}.rekey.epoch")
def _pending_epoch_path(db_path: Path) -> Path:
return db_path.with_name(f"{db_path.name}.rekey.pending")
# Windows sharing violations between the lock-free epoch readers and the
# epoch writers. CPython opens files on Windows without FILE_SHARE_DELETE,
# so while a reader (session validation, on every authenticated request)
# holds ``.rekey.epoch`` / ``.rekey.pending`` open, the writer's
# ``os.replace`` onto it or ``unlink`` of it fails with PermissionError;
# and a reader that opens a file whose replace/unlink is in flight
# (delete-pending) gets PermissionError too. Both windows are a few
# microseconds, so each side retries a PermissionError a bounded number of
# times before giving up. Giving up keeps the pre-existing outcome: the
# reader raises ``RekeyEpochUnreadableError`` (fail closed: no epoch is
# ever returned from a failed read) and the writer re-raises. On POSIX a
# PermissionError here is a genuine permission problem; the retry then
# costs at most ``(attempts - 1) * delay`` before the same outcome.
_SHARING_VIOLATION_ATTEMPTS = 5
_SHARING_VIOLATION_DELAY = 0.01
_T = TypeVar("_T")
def _retry_sharing_violation(operation: Callable[[], _T]) -> _T:
"""Run ``operation``, retrying a transient PermissionError (bounded)."""
for attempt in range(1, _SHARING_VIOLATION_ATTEMPTS + 1):
try:
return operation()
except PermissionError:
if attempt >= _SHARING_VIOLATION_ATTEMPTS:
raise
time.sleep(_SHARING_VIOLATION_DELAY)
raise AssertionError("unreachable") # pragma: no cover
# Every token this module writes is a canonical UUID string: 36 ASCII bytes.
_EPOCH_TOKEN_MAX_BYTES = len(str(UUID(int=0)))
def _read_epoch_file(fd: int) -> bytes:
"""Read at most one byte past a token, so an oversized file is detected."""
data = b""
while len(data) <= _EPOCH_TOKEN_MAX_BYTES:
chunk = os.read(fd, _EPOCH_TOKEN_MAX_BYTES + 1 - len(data))
if not chunk:
break
data += chunk
return data
def _close_epoch_file(fd: int) -> None:
# Read-only descriptor: a failed close loses nothing that was read.
try:
os.close(fd)
except OSError:
return
def _read_epoch_token(path: Path, db_path: Path) -> str | None:
# The guard _refuse_foreign_sidecar gives the lock sidecar, for these
# reads: they run while the caller holds the sidecar flock (every
# authenticated probe does), so a file planted at the token name must
# fail the read, never hang it or fill memory. O_NOFOLLOW refuses a
# symlink (ELOOP). O_NONBLOCK makes the open of a FIFO return at once
# instead of waiting for a writer, and the fstat below refuses every
# file that is not regular (EINVAL); both are "nothing was read". A
# regular file longer than a token is read no further than one byte
# past it, and does not vouch, like any other malformed token. The link
# count is not checked: unlike the sidecar's fchmod, a read-only open
# cannot change the file a hard link points at.
flags = (
os.O_RDONLY
| getattr(os, "O_NOFOLLOW", 0)
| getattr(os, "O_NONBLOCK", 0)
| getattr(os, "O_BINARY", 0)
)
try:
fd = _retry_sharing_violation(lambda: os.open(path, flags))
except FileNotFoundError:
return None
except OSError as error:
raise RekeyEpochUnreadableError(db_path, error.errno) from None
try:
status = os.fstat(fd)
regular = stat.S_ISREG(status.st_mode)
oversized = status.st_size > _EPOCH_TOKEN_MAX_BYTES
raw = _read_epoch_file(fd) if regular and not oversized else b""
except OSError as error:
raise RekeyEpochUnreadableError(db_path, error.errno) from None
finally:
_close_epoch_file(fd)
if not regular:
raise RekeyEpochUnreadableError(db_path, errno.EINVAL)
if oversized or len(raw) > _EPOCH_TOKEN_MAX_BYTES:
raise RekeyRecoveryRequiredError(db_path)
try:
token = raw.decode("ascii")
except UnicodeError:
raise RekeyRecoveryRequiredError(db_path) from None
try:
canonical = str(UUID(token))
except ValueError:
raise RekeyRecoveryRequiredError(db_path) from None
if token != canonical:
raise RekeyRecoveryRequiredError(db_path)
return canonical
def read_rekey_epoch_state(db_path: Path) -> RekeyEpochState:
"""Read and parse the complete durable rekey state.
The application reads this while holding the user's rekey sidecar
lock, which excludes a concurrent publication, with one exception:
``DatabaseManager._classify_rekey_failure`` when ``change_password``
fails after that lock has been released (the read only picks which
error to report). The read order matters for that reader and for the
ones that never take the lock: ``_record_password_verifier`` (only
tests call it) and the ``authorization_epoch`` projection
used by session and socket identity validation. Such a reader may run while
a rekey is publishing, so the two files are read in the order that
cannot tear into a stale-but-stable answer. Writers publish ``.pending`` before
they change ``.epoch`` and remove it only after ``.epoch`` is
committed. Reading ``.epoch`` first could see the OLD committed epoch,
then (after a commit + clear landed in between) no pending file, and
report the old epoch as stable. Reading ``.pending`` first means a
missing pending file was missing BEFORE the epoch read, so the epoch
read is at least as new as that moment; a present one reports the
in-flight state.
"""
pending_epoch = _read_epoch_token(_pending_epoch_path(db_path), db_path)
committed_epoch = _read_epoch_token(_epoch_path(db_path), db_path)
if committed_epoch is None and pending_epoch is not None:
raise RekeyRecoveryRequiredError(db_path)
return RekeyEpochState(committed_epoch, pending_epoch)
def read_rekey_epoch(db_path: Path) -> str | None:
"""Project only a stable committed epoch for cache validation."""
try:
state = read_rekey_epoch_state(db_path)
except RekeyRecoveryRequiredError:
return None
if state.pending_epoch is not None:
return None
return state.committed_epoch
def _canonical_epoch(epoch: str, db_path: Path) -> str:
try:
canonical = str(UUID(epoch))
except ValueError:
raise RekeyRecoveryRequiredError(db_path) from None
if epoch != canonical:
raise RekeyRecoveryRequiredError(db_path)
return canonical
# errnos with which a filesystem says it cannot fsync a directory at all
# (vboxsf, some CIFS/9p mounts, WSL DrvFs). There is nothing stronger to
# fall back to, so the rename stays as durable as that filesystem makes
# it; any other errno (EIO, ENOSPC, EACCES, ...) is a real failure.
_DIRECTORY_FSYNC_UNSUPPORTED: frozenset[int] = frozenset(
code
for code in (
errno.EINVAL,
getattr(errno, "ENOTSUP", None),
getattr(errno, "EOPNOTSUPP", None),
getattr(errno, "ENOSYS", None),
)
if code is not None
)
def _sync_parent_directory(path: Path) -> None:
if os.name == "nt":
return
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
directory_fd = os.open(path.parent, flags)
try:
os.fsync(directory_fd)
except OSError as error:
# Unsupported, not failed: treating it as fatal made every epoch
# publication fail on such mounts (the first login after upgrade,
# every password change, every registration).
if error.errno not in _DIRECTORY_FSYNC_UNSUPPORTED:
raise
finally:
os.close(directory_fd)
# Allow interrupted writers ample time before treating their temps as stale.
_REKEY_TEMP_GRACE_SECONDS = 60 * 60
def cleanup_stale_rekey_temps(db_path: Path) -> None:
"""Best-effort crash-temp cleanup; caller must hold the exclusive sidecar.
Match the writer's literal dot-prefixed names, never a glob derived from
the database basename. No file is opened: in particular, a planted hard
link must not release SQLite's POSIX locks by opening/closing its inode.
"""
prefixes = (
f".{db_path.name}.rekey.epoch.",
f".{db_path.name}.rekey.pending.",
)
cutoff = time.time() - _REKEY_TEMP_GRACE_SECONDS
try:
paths = db_path.parent.iterdir()
for path in paths:
if not path.name.startswith(prefixes):
continue
# CPython mkstemp appends eight lowercase ASCII letters, digits
# or underscores. Be conservative if that format ever changes;
# a longer name can belong to another database or a backup.
suffix = path.name[
len(next(p for p in prefixes if path.name.startswith(p))) :
]
if len(suffix) != 8 or any(
char not in "abcdefghijklmnopqrstuvwxyz0123456789_"
for char in suffix
):
continue
try:
status = path.lstat()
if (
stat.S_ISREG(status.st_mode)
and status.st_nlink == 1
and status.st_mtime < cutoff
):
path.unlink()
except OSError:
# Cosmetic leftovers must not make login unavailable.
continue
except OSError:
# Listing the directory is cosmetic too; sidecar errors stay typed.
return
def _write_epoch_token(path: Path, epoch: str, db_path: Path) -> None:
canonical = _canonical_epoch(epoch, db_path)
# The temporary file materialized below holds only a random canonical
# UUID epoch token (no secrets, no user data), is created 0o600 by
# mkstemp inside the database's own encrypted-databases directory, and
# is unlinked in the finally below whether or not the replace lands.
fd, temporary_name = tempfile.mkstemp( # Safe: UUID epoch token only
dir=path.parent,
prefix=f".{path.name}.",
)
temporary_path = Path(temporary_name)
try:
# Same canonical UUID token, nothing else, before fsync+replace.
stream = os.fdopen(fd, "wb") # Safe: UUID epoch token only
fd = -1
with stream:
_ = stream.write(canonical.encode("ascii"))
stream.flush()
os.fsync(stream.fileno())
# A lock-free reader holding ``path`` open makes this fail with a
# sharing violation on Windows: retry it (bounded), see
# ``_retry_sharing_violation``.
# nosemgrep: semgrep.rules.path-traversal-risk, reason: path is a fixed suffix on the sha256-named user database path and temporary_path is mkstemp output in its directory
_retry_sharing_violation(lambda: os.replace(temporary_path, path))
_sync_parent_directory(path)
finally:
if fd >= 0:
os.close(fd)
# nosemgrep: semgrep.rules.path-traversal-risk, reason: temporary_path is mkstemp output in the directory of the sha256-named user database path
temporary_path.unlink(missing_ok=True)
def begin_rekey_epoch(db_path: Path) -> str:
"""Durably publish a fresh pending generation from stable state."""
state = read_rekey_epoch_state(db_path)
if state.committed_epoch is None or state.pending_epoch is not None:
raise RekeyRecoveryRequiredError(db_path)
pending_epoch = str(uuid4())
while pending_epoch == state.committed_epoch:
pending_epoch = str(uuid4())
_write_epoch_token(_pending_epoch_path(db_path), pending_epoch, db_path)
return pending_epoch
def commit_rekey_epoch(db_path: Path, pending_epoch: str) -> None:
"""Durably publish the verified pending generation as committed."""
candidate = _canonical_epoch(pending_epoch, db_path)
state = read_rekey_epoch_state(db_path)
if state.committed_epoch is None or state.pending_epoch != candidate:
raise RekeyRecoveryRequiredError(db_path)
if state.committed_epoch != candidate:
_write_epoch_token(_epoch_path(db_path), candidate, db_path)
def clear_pending_rekey_epoch(db_path: Path, pending_epoch: str) -> None:
"""Remove pending metadata only after its generation is committed."""
candidate = _canonical_epoch(pending_epoch, db_path)
state = read_rekey_epoch_state(db_path)
if state.pending_epoch is None and state.committed_epoch == candidate:
return
if state.committed_epoch != candidate or state.pending_epoch != candidate:
raise RekeyRecoveryRequiredError(db_path)
pending_path = _pending_epoch_path(db_path)
# Same Windows sharing violation as the replace in _write_epoch_token.
_retry_sharing_violation(pending_path.unlink)
_sync_parent_directory(pending_path)
def advance_rekey_epoch(db_path: Path) -> str:
"""Atomically publish a fresh durable rekey epoch."""
state = read_rekey_epoch_state(db_path)
if state.pending_epoch is not None:
raise RekeyRecoveryRequiredError(db_path)
epoch = str(uuid4())
while epoch == state.committed_epoch:
epoch = str(uuid4())
_write_epoch_token(_epoch_path(db_path), epoch, db_path)
return epoch
def publish_new_database_epoch(db_path: Path) -> str:
"""Mint a fresh committed epoch for a database that was just created.
Creation proved no database existed at this path, so any epoch or
pending token left beside it (an account removed without its sidecars,
a restore of only the ``.db``) belongs to an earlier database. Reusing
that epoch would let an engine or identity cached under the earlier
database vouch for the new one, and a leftover pending token would
make the creation fail. The fresh epoch is published first and the
stale pending token removed after it, so a reader never sees the old
epoch as stable.
"""
try:
previous = _read_epoch_token(_epoch_path(db_path), db_path)
except RekeyRecoveryRequiredError:
previous = None
epoch = str(uuid4())
while epoch == previous:
epoch = str(uuid4())
_write_epoch_token(_epoch_path(db_path), epoch, db_path)
pending_path = _pending_epoch_path(db_path)
try:
# Same Windows sharing violation as clear_pending_rekey_epoch's
# unlink: a lock-free reader holding the stale pending token open
# must not fail the creation. FileNotFoundError is not retried.
_retry_sharing_violation(pending_path.unlink)
except FileNotFoundError:
return epoch
_sync_parent_directory(pending_path)
return epoch
def read_or_initialize_rekey_epoch(db_path: Path) -> str:
"""Read or initialize the epoch while the stable sidecar lock is held."""
state = read_rekey_epoch_state(db_path)
if state.pending_epoch is not None:
raise RekeyRecoveryRequiredError(db_path)
if state.committed_epoch is not None:
return state.committed_epoch
return advance_rekey_epoch(db_path)