Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.4 KB
Raw Blame
"""Regression test: every shipped default must pass the REAL save-path
validation it would face if a user hit "Save" without changing it.

Background (see fix commit "the shipped default theme named a theme that
no longer exists"): ``app.theme`` shipped as ``"dark"`` while
``settings/manager.py`` replaces that setting's ``options`` at runtime from
the theme registry, and the generated registry list had no ``dark`` entry.
The shipped default named a value that was not among its own runtime
options. This stayed invisible because seeding a fresh install goes through
``manager.py``'s ``validate_imported_setting_value``, which has a guard
that vouches for a value *because* it equals the shipped default — so an
invalid default seeds successfully. The only path that ever actually
validated the stored value against its live constraints was the
JavaScript-disabled Save path (``coerce_setting_for_write`` +
``validate_setting`` in ``web/routers/settings.py``), which then reported a
user's completely untouched theme selection as a validation failure.

This test closes that blind spot for every setting, not just
``app.theme``. It does not reimplement the validation rules — it calls the
app's own ``coerce_setting_for_write`` / ``validate_setting`` functions,
exactly as the Save route does, against every value shipped in
``default_settings.json`` (as resolved by ``SettingsManager``, which is
what injects the *runtime* options for the handful of settings whose
option lists are generated from code rather than read verbatim from the
JSON — e.g. ``app.theme`` from the theme registry and
``search.search_strategy`` from ``constants.get_available_strategies``).

The assertion is a hard zero, not a count. A count that can be edited
upward is exactly the shape of guard that let the ``app.theme`` bug survive
undetected for as long as it did. Any setting that is legitimately exempt
must be named explicitly in ``EXEMPT_KEYS`` with a reason — never absorbed
by loosening the assertion.
"""

import os

import pytest

from local_deep_research.database.models import Setting
from local_deep_research.settings.manager import (
    SettingsManager,
    _filter_setting_columns,
)
from local_deep_research.web.routers.settings import (
    coerce_setting_for_write,
    validate_setting,
)

# ---------------------------------------------------------------------------
# Exemptions: settings whose shipped default is known, on inspection, to be
# intentional/correct product behaviour even though the real validation
# pipeline currently rejects it. Every entry names *why*, so this list can
# only grow by a deliberate, reviewable decision -- never by weakening the
# assertion below.
# ---------------------------------------------------------------------------

EXEMPT_KEYS: dict[str, str] = {
    # Empty on purpose.
    #
    # This started with one entry: ``embeddings.openai.dimensions``, an
    # optional numeric whose correct shipped default is ``null``. It was NOT a
    # bad default -- the description says "Leave blank to use the model's
    # native dimensionality" -- it was a real second instance of the app.theme
    # bug class, one layer down: ``validate_setting`` required
    # ``isinstance(value, (int, float))`` unconditionally for number/slider/
    # range, so the application rejected a value it ships and a user who left
    # the field blank was told "Value must be a number" by the JS-disabled Save
    # path. Fixed in ``web/routers/settings.py`` by treating ``None`` as the
    # unset state it is (the column is nullable and no "required" flag exists
    # anywhere in the Setting model), so the exemption is gone rather than
    # carried.
    #
    # If you add an entry here, give it a reason and a TODO. An exemption list
    # that grows silently is how the original defect survived: the guard in
    # settings/manager.py that accepts "the value equals the shipped default"
    # meant an invalid default seeded cleanly and was only visible from a path
    # that actually validated it.
}


@pytest.fixture(autouse=True)
def _clean_ldr_env():
    """Save/clear/restore LDR_* env vars so an env override can't mask (or
    fake) a validation failure -- mirrors the fixture in
    test_settings_defaults_integrity.py."""
    original_env = {k: v for k, v in os.environ.items() if k.startswith("LDR_")}
    for key in list(os.environ.keys()):
        if key.startswith("LDR_"):
            os.environ.pop(key, None)
    yield
    for key in list(os.environ.keys()):
        if key.startswith("LDR_"):
            os.environ.pop(key, None)
    for key, value in original_env.items():
        os.environ[key] = value


def _resolved_defaults() -> dict:
    """Defaults as SettingsManager actually computes them at runtime,
    including the dynamic options injection (theme registry,
    search-strategy list, ...) done by `SettingsManager.default_settings`.
    """
    manager = SettingsManager(db_session=None)
    return manager.default_settings


def _setting_row(key: str, meta: dict) -> Setting:
    """Build the same `Setting` ORM object the app itself builds for a
    no-DB read of this key (see `SettingsManager.__query_settings`), so
    validation runs against the exact object shape `validate_setting`
    expects -- not a hand-rolled stand-in.
    """
    return Setting(key=key, **_filter_setting_columns(meta))


def test_every_shipped_default_passes_real_save_path_validation():
    """No shipped default may fail the real `coerce_setting_for_write` +
    `validate_setting` pipeline -- the exact pipeline the no-JS Save route
    runs on every setting, changed or not.

    This must never be relaxed to "no more than N offenders" -- see module
    docstring. Zero, or an explicitly named+reasoned exemption above.
    """
    defaults = _resolved_defaults()

    # Guard the guard: if this collapses to ~0 keys, `default_settings`
    # failed to load and the test below would vacuously pass.
    assert len(defaults) >= 400, (
        f"Only {len(defaults)} default settings loaded; "
        "SettingsManager.default_settings may have failed to load the "
        "defaults JSON files -- the check below would be meaningless."
    )

    offenders = []
    for key, meta in sorted(defaults.items()):
        if key in EXEMPT_KEYS:
            continue

        ui_element = meta.get("ui_element", "text")
        value = meta.get("value")
        row = _setting_row(key, meta)

        try:
            converted_value = coerce_setting_for_write(
                key=key, value=value, ui_element=ui_element
            )
            is_valid, error_message = validate_setting(row, converted_value)
        except Exception as exc:  # noqa: BLE001 - report, don't hide
            is_valid, error_message = False, f"raised {exc!r}"

        if not is_valid:
            offenders.append(
                f"{key} (ui_element={ui_element!r}, value={value!r}): "
                f"{error_message}"
            )

    assert not offenders, (
        f"{len(offenders)} shipped default(s) fail real runtime validation "
        "-- a user who never touches these settings and hits Save (e.g. "
        "the JS-disabled Save path) would have them rejected:\n"
        + "\n".join(f"  {line}" for line in offenders)
    )


def test_exempt_keys_are_still_present_and_still_exempt_for_stated_reason():
    """Guards against the exemption list going stale: every exempted key
    must still exist in defaults, and must still actually fail validation
    for the stated reason. If a future fix makes it pass, the exemption
    must be deleted, not left as dead weight (and if it starts failing for
    a *different* reason, that's a new bug to look at, not the old one).
    """
    defaults = _resolved_defaults()

    for key, reason in EXEMPT_KEYS.items():
        assert key in defaults, (
            f"Exempted key {key!r} no longer exists in defaults -- remove "
            f"its EXEMPT_KEYS entry (reason was: {reason})"
        )

        meta = defaults[key]
        ui_element = meta.get("ui_element", "text")
        value = meta.get("value")
        row = _setting_row(key, meta)

        converted_value = coerce_setting_for_write(
            key=key, value=value, ui_element=ui_element
        )
        is_valid, _error_message = validate_setting(row, converted_value)

        assert not is_valid, (
            f"Exempted key {key!r} now PASSES real validation -- the "
            f"underlying issue was fixed. Delete its EXEMPT_KEYS entry so "
            f"the zero-offenders assertion covers it again. "
            f"(stated reason was: {reason})"
        )