Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.0 KB
Raw Blame
"""Per-database salt files and Windows text-mode newline translation.

On Windows, ``os.open`` without ``O_BINARY`` returns a text-mode
descriptor, and ``os.write`` then stores a CR (0x0D) before every LF
(0x0A). A random 32-byte salt contains an LF with probability
1 - (255/256)**32, about 12%, so such a salt file was stored with 33 or
more bytes and ``get_salt_for_database`` refused it as corrupted.

These tests pin both halves of the fix:

* ``create_database_salt`` passes ``O_BINARY`` wherever the platform
  defines it;
* ``get_salt_for_database`` reads a file damaged this way back as the
  original salt, never rewrites it, and still refuses every other size.
"""

import os
import random
from types import SimpleNamespace

import pytest
from sqlalchemy import text

from local_deep_research.database import sqlcipher_utils
from local_deep_research.database.sqlcipher_utils import (
    SALT_SIZE,
    create_database_salt,
    get_salt_file_path,
    get_salt_for_database,
)


def _text_mode_write(salt: bytes) -> bytes:
    """The bytes a Windows text-mode write stores for *salt*."""
    return salt.replace(b"\n", b"\r\n")


def _warnings(caplog):
    return [
        r
        for r in caplog.records
        if r.levelname in ("WARNING", "ERROR", "CRITICAL")
    ]


# 32-byte salts containing LF, including CR LF and CR CR LF runs whose CR
# bytes belong to the salt rather than being inserted by the write.
SALTS_WITH_LF = [
    pytest.param(b"\n" + b"\x01" * 31, id="leading-lf"),
    pytest.param(b"\x02" * 31 + b"\n", id="trailing-lf"),
    pytest.param(b"\x03" * 10 + b"\r\n" + b"\x04" * 20, id="crlf-run"),
    pytest.param(b"\x05" * 9 + b"\r\r\n" + b"\x06" * 20, id="crcrlf-run"),
    pytest.param(b"\r\n\n\r\r\n\r" + b"\x07" * 25, id="mixed-runs"),
    pytest.param(b"\n" * SALT_SIZE, id="all-lf"),
]


def test_create_database_salt_opens_the_salt_file_with_o_binary(
    tmp_path, monkeypatch
):
    """The writer must request binary mode wherever the platform has it.

    ``os.O_BINARY`` does not exist on POSIX, so a sentinel bit stands in
    for it. The wrapper records the flags and swaps the sentinel back for
    the real flag (0 here) before the real call.
    """
    real_open = os.open
    real_binary = getattr(os, "O_BINARY", 0)
    sentinel = 1 << 30
    assert not sentinel & (os.O_WRONLY | os.O_CREAT | os.O_EXCL | real_binary)
    monkeypatch.setattr(os, "O_BINARY", sentinel, raising=False)

    calls = []

    def recording_open(path, flags, *args, **kwargs):
        calls.append((os.fspath(path), flags))
        if flags & sentinel:
            flags = (flags & ~sentinel) | real_binary
        return real_open(path, flags, *args, **kwargs)

    monkeypatch.setattr(os, "open", recording_open)

    db_path = tmp_path / "user.db"
    salt = create_database_salt(db_path)

    salt_file = get_salt_file_path(db_path)
    salt_opens = [flags for path, flags in calls if path == str(salt_file)]
    assert len(salt_opens) == 1
    assert salt_opens[0] & sentinel, "the salt file was opened without O_BINARY"
    assert salt_file.read_bytes() == salt


@pytest.mark.parametrize("salt", SALTS_WITH_LF)
def test_a_text_mode_damaged_salt_reads_back_as_the_original(tmp_path, salt):
    assert len(salt) == SALT_SIZE
    db_path = tmp_path / "user.db"
    salt_file = get_salt_file_path(db_path)
    damaged = _text_mode_write(salt)
    salt_file.write_bytes(damaged)

    assert get_salt_for_database(db_path) == salt
    # No healing write: the damaged file stays exactly as it was.
    assert salt_file.read_bytes() == damaged


def test_recovery_is_exact_for_salts_made_of_cr_and_lf_runs(tmp_path):
    """Class check over salts drawn mostly from CR and LF bytes, so that
    every run shape (LF LF, CR LF, CR CR LF, a trailing CR, ...) occurs."""
    rng = random.Random(1439)
    alphabet = b"\r\n\r\n\x00\xff"
    db_path = tmp_path / "user.db"
    salt_file = get_salt_file_path(db_path)
    for _ in range(500):
        salt = bytes(rng.choice(alphabet) for _ in range(SALT_SIZE))
        salt_file.write_bytes(_text_mode_write(salt))
        assert get_salt_for_database(db_path) == salt


def test_recovery_warns_once_naming_the_file_but_not_the_bytes(
    tmp_path, loguru_caplog
):
    salt = b"\x08" * 15 + b"\n" + b"\x09" * 15 + b"\r"
    damaged = _text_mode_write(salt)
    db_path = tmp_path / "user.db"
    salt_file = get_salt_file_path(db_path)
    salt_file.write_bytes(damaged)

    with loguru_caplog.at_level("WARNING"):
        assert get_salt_for_database(db_path) == salt

    warnings = _warnings(loguru_caplog)
    assert len(warnings) == 1
    assert str(salt_file) in warnings[0].getMessage()
    logged = loguru_caplog.text
    for form in (salt, damaged):
        assert form.hex() not in logged
        assert form.hex().upper() not in logged
        assert repr(form) not in logged


@pytest.mark.parametrize(
    "salt",
    [
        pytest.param(b"\r\n" * 16, id="all-crlf"),
        pytest.param(b"A" * 15 + b"\r\n" + b"B" * 15, id="one-crlf"),
        pytest.param(b"\r\r\n" + b"C" * 29, id="crcrlf"),
    ],
)
def test_an_intact_salt_containing_crlf_is_returned_unchanged(
    tmp_path, salt, loguru_caplog
):
    """A file of exactly SALT_SIZE bytes is never translated: CR LF in a
    32-byte salt is salt data, not damage."""
    assert len(salt) == SALT_SIZE
    db_path = tmp_path / "user.db"
    get_salt_file_path(db_path).write_bytes(salt)

    with loguru_caplog.at_level("WARNING"):
        assert get_salt_for_database(db_path) == salt
    assert not _warnings(loguru_caplog)


@pytest.mark.parametrize(
    "content",
    [
        pytest.param(b"A" * 33, id="33-bytes-no-crlf"),
        # Collapsing CR LF gives 32 bytes, but the bare LF shows that this
        # is not what a text-mode write produces.
        pytest.param(
            b"B" * 30 + b"\r\n" + b"\n", id="33-bytes-crlf-and-bare-lf"
        ),
        pytest.param(b"C" * 31, id="31-bytes"),
        pytest.param(b"D" * 29 + b"\r\n", id="31-bytes-with-crlf"),
        # Only a single translation is undone.
        pytest.param(
            _text_mode_write(_text_mode_write(b"\n" + b"E" * 31)),
            id="translated-twice",
        ),
        pytest.param(b"", id="empty"),
    ],
)
def test_any_other_wrong_size_still_raises(tmp_path, content):
    db_path = tmp_path / "user.db"
    get_salt_file_path(db_path).write_bytes(content)

    with pytest.raises(
        ValueError, match=rf"unexpected size \({len(content)} bytes\)"
    ) as exc_info:
        get_salt_for_database(db_path)
    assert "may be corrupted" in str(exc_info.value)


def test_a_damaged_salt_file_still_opens_its_database(tmp_path, monkeypatch):
    """End to end: a real SQLCipher database whose salt contains LF bytes,
    with its salt file rewritten the way a Windows text-mode write stores
    it, opens with the right password through the normal open path."""
    pytest.importorskip(
        "sqlcipher3", reason="requires SQLCipher (encrypted mode)"
    )
    from local_deep_research.database.encrypted_db import DatabaseManager

    monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))
    monkeypatch.setenv("LDR_DB_CONFIG_KDF_ITERATIONS", "1000")
    manager = DatabaseManager()
    manager.data_dir = tmp_path
    try:
        if not manager.has_encryption:
            pytest.skip("requires SQLCipher (encrypted mode)")

        salt = (
            b"\n"
            + b"\x20" * 9
            + b"\r\n"
            + b"\x21" * 9
            + b"\r\r\n"
            + b"\x22" * 8
        )
        assert len(salt) == SALT_SIZE

        def fixed_token_bytes(nbytes):
            assert nbytes == SALT_SIZE
            return salt

        username = "salt_text_mode_user"
        password = "SaltTextModePassword1!"  # noqa: S105
        with monkeypatch.context() as patch:
            patch.setattr(
                sqlcipher_utils,
                "secrets",
                SimpleNamespace(token_bytes=fixed_token_bytes),
            )
            engine = manager.create_user_database(username, password)
        with engine.begin() as conn:
            conn.execute(
                text(
                    "CREATE TABLE salt_canary (id INTEGER PRIMARY KEY, note TEXT)"
                )
            )
            conn.execute(
                text("INSERT INTO salt_canary (id, note) VALUES (1, 'intact')")
            )
        manager.close_user_database(username)

        salt_file = get_salt_file_path(manager._get_user_db_path(username))
        assert salt_file.read_bytes() == salt
        damaged = _text_mode_write(salt)
        salt_file.write_bytes(damaged)

        reopened = manager.open_user_database(username, password)
        assert reopened is not None, "the damaged salt file locked the user out"
        with reopened.connect() as conn:
            assert (
                conn.execute(
                    text("SELECT note FROM salt_canary WHERE id = 1")
                ).scalar()
                == "intact"
            )
        manager.close_user_database(username)

        assert manager.open_user_database(username, "Wrong-Password-1!") is None
        assert salt_file.read_bytes() == damaged
    finally:
        manager.close_all_databases()