Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.8 KB
Raw Blame
"""Crash leftovers are cosmetic; sweep them only during a locked cold open."""

import multiprocessing
import os
from pathlib import Path
from uuid import uuid4

import pytest
from sqlalchemy import text

from local_deep_research.database import rekey_lock
from local_deep_research.database.encrypted_db import DatabaseManager


def _crash_writer(db, kind, ready, release):
    path = Path(db)

    def pause_replace(source, destination):
        ready.send(str(source))
        release.wait(30)
        raise RuntimeError("parent did not kill writer")

    rekey_lock.os.replace = pause_replace
    with rekey_lock.acquire_rekey_lock(path):
        rekey_lock._write_epoch_token(
            path.with_name(f"{path.name}.rekey.{kind}"), str(uuid4()), path
        )


def _orphan(db, kind):
    ctx = multiprocessing.get_context("spawn")
    receiver, sender = ctx.Pipe(duplex=False)
    release = ctx.Event()
    child = ctx.Process(
        target=_crash_writer, args=(str(db), kind, sender, release)
    )
    child.start()
    sender.close()
    try:
        assert receiver.poll(20), "writer never reached replace"
        path = Path(receiver.recv())
        assert path.is_file()
        assert path.read_text()  # real flushed production token
        child.kill()
        child.join(10)
        assert not child.is_alive()
        assert child.exitcode != 0
        assert path.is_file(), "kill must bypass writer's finally unlink"
        return path
    finally:
        if child.is_alive():
            child.kill()
            child.join(10)
        receiver.close()
        child.close()


@pytest.fixture
def native_db(tmp_path, monkeypatch):
    monkeypatch.setenv("LDR_DB_KDF_ITERATIONS", "1000")
    monkeypatch.setenv("LDR_ALLOW_UNENCRYPTED", "false")
    manager = DatabaseManager()
    assert manager.has_encryption
    manager.data_dir = tmp_path
    # Phase two concerns vector stores, not the main encrypted database.
    monkeypatch.setattr(manager, "_run_phase2_rekey", lambda *args: None)
    engine = manager.create_user_database("alice", "TestPass1!")
    with engine.begin() as conn:
        assert conn.exec_driver_sql("PRAGMA cipher_version").scalar()
        conn.execute(text("CREATE TABLE cleanup_probe (value TEXT)"))
        conn.execute(text("INSERT INTO cleanup_probe VALUES ('preserved')"))
    manager.close_user_database("alice")
    yield manager, manager._get_user_db_path("alice")
    manager.close_all_databases()


def _open_preserved(manager):
    engine = manager.open_user_database("alice", "TestPass1!")
    assert engine is not None
    with engine.connect() as conn:
        assert (
            conn.execute(text("SELECT value FROM cleanup_probe")).scalar()
            == "preserved"
        )
    return engine


def test_cold_open_removes_real_crash_orphans(native_db):
    manager, db = native_db
    leftovers = [_orphan(db, kind) for kind in ("epoch", "pending")]
    for path in leftovers:
        assert path.name.startswith(f".{db.name}.rekey.")
        os.utime(path, (1, 1))
    hardlink = db.with_name(f".{db.name}.rekey.epoch.hardlink")
    hardlink.hardlink_to(db)
    symlink = db.with_name(f".{db.name}.rekey.pending.symlink0")
    symlink.symlink_to(db)
    os.utime(hardlink, (1, 1))
    _open_preserved(manager)
    assert all(not path.exists() for path in leftovers), (
        "stale crash temps survived cold open"
    )
    assert hardlink.stat().st_ino == db.stat().st_ino
    assert symlink.is_symlink()
    manager.close_user_database("alice")
    _open_preserved(manager)


def _temp(db, suffix="epoch.abcdefgh", mtime=1):
    path = db.with_name(f".{db.name}.rekey.{suffix}")
    path.write_text("cosmetic", encoding="ascii")
    os.utime(path, (mtime, mtime))
    return path


@pytest.mark.parametrize("kind", ["epoch", "pending"])
def test_literal_names_age_and_special_file_preservation(
    tmp_path, monkeypatch, kind
):
    db = tmp_path / ("user[ab].db" if os.name == "nt" else "user[ab]*?.db")
    now = 10000
    monkeypatch.setattr(rekey_lock.time, "time", lambda: now)
    prefix = f".{db.name}.rekey.{kind}."
    stale = _temp(db, f"{kind}.stale000", now - 3600 - 1)
    keep = [
        _temp(db, f"{kind}.boundary", now - 3600),
        _temp(db, f"{kind}.fresh000", now - 3599),
        _temp(db, f"{kind}.future00", now + 1),
        _temp(db, f"{kind}.", 1),
        _temp(
            tmp_path / ("usera.db" if os.name == "nt" else "useraa.db"),
            f"{kind}.other000",
        ),
        _temp(tmp_path / f"{db.name}.other", f"{kind}.other000"),
    ]
    # The database and every durable/non-temp metadata name are outside scope.
    for suffix in (
        "",
        "-wal",
        "-shm",
        ".salt",
        "-journal",
        ".backup",
        ".rekey.epoch",
        ".rekey.pending",
        ".rekey.lock",
    ):
        path = db.with_name(db.name + suffix)
        path.write_text("preserve", encoding="ascii")
        os.utime(path, (1, 1))
        keep.append(path)
    target = tmp_path / "target"
    target.write_text("target", encoding="ascii")
    symlink_target = tmp_path / "symlink-target"
    symlink_target.write_text("symlink target", encoding="ascii")
    os.utime(symlink_target, (1, 1))
    symlink = tmp_path / (prefix + "symlink0")
    symlink.symlink_to(symlink_target)
    if os.utime in os.supports_follow_symlinks:
        os.utime(symlink, (1, 1), follow_symlinks=False)
    hardlink = tmp_path / (prefix + "hardlink")
    hardlink.hardlink_to(target)
    os.utime(target, (1, 1))
    directory = tmp_path / (prefix + "director")
    directory.mkdir()
    fifo = tmp_path / (prefix + "fifo0000")
    if hasattr(os, "mkfifo"):
        os.mkfifo(fifo)
        os.utime(fifo, (1, 1))
    os.utime(directory, (1, 1))
    nested_other = _temp(tmp_path / f"{db.name}.rekey.{kind}.otherdb0")
    invalid_suffix = _temp(db, f"{kind}.INVALID!")
    backup = _temp(db, f"{kind}.abcdefgh.backup")
    with rekey_lock.acquire_rekey_lock(db):
        rekey_lock.cleanup_stale_rekey_temps(db)
    assert not stale.exists()
    assert nested_other.exists()
    assert invalid_suffix.exists()
    assert backup.exists()
    assert all(path.exists() for path in keep)
    assert symlink.is_symlink()
    assert hardlink.stat().st_nlink == 2
    assert directory.is_dir()
    if hasattr(os, "mkfifo"):
        assert fifo.exists()
    assert target.read_text() == "target"
    assert symlink_target.read_text() == "symlink target"
    assert all(path.read_text() == "preserve" for path in keep[6:])


def _hold_lock(db, ready, release):
    with rekey_lock.acquire_rekey_lock(Path(db)):
        ready.send("acquired")
        if not release.wait(30):
            raise RuntimeError("parent did not release holder")


def test_cold_open_excluded_by_foreign_lock_then_cleans(native_db):
    manager, db = native_db
    stale = _temp(db)
    ctx = multiprocessing.get_context("spawn")
    receiver, sender = ctx.Pipe(duplex=False)
    release = ctx.Event()
    child = ctx.Process(target=_hold_lock, args=(str(db), sender, release))
    child.start()
    sender.close()
    try:
        assert receiver.poll(20)
        assert receiver.recv() == "acquired"
        with pytest.raises(rekey_lock.RekeyLockContentionError):
            manager.open_user_database("alice", "TestPass1!")
        assert stale.exists()
        release.set()
        child.join(10)
        assert not child.is_alive()
        assert child.exitcode == 0
        _open_preserved(manager)
        assert not stale.exists()
        manager.close_user_database("alice")
        _open_preserved(manager)
    finally:
        release.set()
        if child.is_alive():
            child.kill()
            child.join(10)
        receiver.close()
        child.close()


def test_cache_and_probes_never_scan_cold_open_holds_sidecar(
    native_db, monkeypatch
):
    manager, db = native_db
    engine = _open_preserved(manager)
    stale = _temp(db)
    scans = []
    real_iterdir = Path.iterdir

    def observe(directory):
        if directory == db.parent:
            scans.append(directory)
            # A separate process must actually contend during the scan.
            ctx = multiprocessing.get_context("spawn")
            receiver, sender = ctx.Pipe(duplex=False)
            child = ctx.Process(target=_try_lock, args=(str(db), sender))
            child.start()
            sender.close()
            try:
                assert receiver.poll(20)
                outcome = receiver.recv()
                child.join(10)
                assert not child.is_alive()
                assert child.exitcode == 0
                scans.append(outcome)
            finally:
                if child.is_alive():
                    child.kill()
                    child.join(10)
                receiver.close()
                child.close()
        return real_iterdir(directory)

    monkeypatch.setattr(Path, "iterdir", observe)
    assert manager.open_user_database("alice", "TestPass1!") is engine
    assert manager.is_user_connected("alice")
    assert manager._password_matches_cached("alice", "TestPass1!")
    assert scans == []
    assert stale.exists()
    manager.close_user_database("alice")
    _open_preserved(manager)
    # Assertions outside the cleanup's best-effort exception handlers.
    assert scans == [db.parent, "contended"]
    assert not stale.exists()


def _try_lock(db, ready):
    try:
        with rekey_lock.acquire_rekey_lock(Path(db)):
            result = "unexpected acquisition"
    except rekey_lock.RekeyLockContentionError:
        result = "contended"
    ready.send(result)


@pytest.mark.parametrize("operation", ["list", "lstat", "unlink", "disappear"])
def test_cosmetic_io_failure_still_opens_encrypted_data(
    native_db, monkeypatch, operation
):
    manager, db = native_db
    stale = _temp(db)
    reached = []
    real_iterdir, real_lstat, real_unlink = (
        Path.iterdir,
        Path.lstat,
        Path.unlink,
    )

    def failing_list(path):
        if path == db.parent:
            reached.append("list")
            raise PermissionError("injected list denial")
        return real_iterdir(path)

    def failing_lstat(path):
        if path == stale:
            reached.append("lstat")
            raise PermissionError("injected lstat denial")
        return real_lstat(path)

    def failing_unlink(path, *args, **kwargs):
        if path == stale:
            reached.append(operation)
            if operation == "disappear":
                real_unlink(path)
                raise FileNotFoundError("entry disappeared")
            raise PermissionError("injected unlink denial")
        return real_unlink(path, *args, **kwargs)

    with monkeypatch.context() as patch:
        if operation == "list":
            patch.setattr(Path, "iterdir", failing_list)
        elif operation == "lstat":
            patch.setattr(Path, "lstat", failing_lstat)
        else:
            patch.setattr(Path, "unlink", failing_unlink)
        _open_preserved(manager)
    assert reached == [operation]
    assert stale.exists() == (operation != "disappear")
    manager.close_user_database("alice")
    _open_preserved(manager)
    assert not stale.exists()


def test_sidecar_operational_failure_still_propagates(native_db, monkeypatch):
    manager, db = native_db
    stale = _temp(db)
    reached = []
    real_open = rekey_lock.os.open

    def denied(path, *args, **kwargs):
        if Path(path) == db.with_name(f"{db.name}.rekey.lock"):
            reached.append(path)
            raise PermissionError(13, "injected sidecar denial")
        return real_open(path, *args, **kwargs)

    with monkeypatch.context() as patch:
        patch.setattr(rekey_lock.os, "open", denied)
        with pytest.raises(rekey_lock.RekeySidecarIOError) as caught:
            manager.open_user_database("alice", "TestPass1!")
    assert reached
    assert caught.value.operation == rekey_lock.RekeySidecarOperation.OPEN
    assert stale.exists()
    _open_preserved(manager)
    assert not stale.exists()


@pytest.mark.nonroot
def test_real_unlink_permission_denial_is_cosmetic(native_db, monkeypatch):
    if os.name == "nt" or os.geteuid() == 0:
        pytest.skip("requires POSIX permission enforcement")
    manager, db = native_db
    stale = _temp(db)
    real_unlink = Path.unlink
    denials = []

    def protected_unlink(path, *args, **kwargs):
        if path != stale:
            return real_unlink(path, *args, **kwargs)
        original_mode = db.parent.stat().st_mode & 0o777
        db.parent.chmod(0o500)
        try:
            try:
                return real_unlink(path, *args, **kwargs)
            except PermissionError as error:
                denials.append(error.errno)
                raise
        finally:
            db.parent.chmod(original_mode)

    with monkeypatch.context() as patch:
        patch.setattr(Path, "unlink", protected_unlink)
        _open_preserved(manager)
    assert denials == [13]
    assert stale.exists()
    manager.close_user_database("alice")
    _open_preserved(manager)
    assert not stale.exists()