Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

95.5 KB
Raw Blame
import functools
import json
import math
import os
import string
import threading
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union

from loguru import logger
from sqlalchemy import and_, func, or_
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session

from .. import defaults
from ..__version__ import __version__ as package_version
from ..database.models import Setting, SettingType
from ..utilities.sql_utils import escape_like
from ..utilities.type_utils import to_bool, to_positive_int
from ..web.models.settings import (
    AppSetting,
    BaseSetting,
    ChatSetting,
    LLMSetting,
    ReportSetting,
    SearchSetting,
)
from .base import ISettingsManager
from .env_registry import registry as env_registry
from .exceptions import SettingsValueError


# Existing saved values remain in place, but accounts cannot redirect library
# reads or writes through their settings after this release.
OPERATOR_ONLY_SETTINGS = frozenset({"research_library.storage_path"})

# SQLite's LIKE ignores case for ASCII letters only. get_setting() finds child
# rows with LIKE, so "RESEARCH_LIBRARY.STORAGE_PATH.x" is a child of
# "research_library.storage_path" there.
_ASCII_CASE_FOLD = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)


def is_operator_only_setting(key: str) -> bool:
    """Whether only trusted defaults or an LDR_* environment override may set key.

    Matches each operator-only key and its dotted child keys, ignoring ASCII
    case the same way SQLite's LIKE does in the child-row lookup.
    """
    folded = key.translate(_ASCII_CASE_FOLD)
    for root in OPERATOR_ONLY_SETTINGS:
        folded_root = root.translate(_ASCII_CASE_FOLD)
        if folded == folded_root or folded.startswith(f"{folded_root}."):
            return True
    return False


def parse_boolean(value: Any) -> bool:
    """
    Convert various representations to boolean using HTML checkbox semantics.

    This function handles form values, JSON booleans, and environment variables,
    ensuring consistent behavior across client and server.

    **HTML Checkbox Semantics** (INTENTIONAL DESIGN):
    - **Any value present (except explicit false) = checked = True**
    - This matches standard HTML form behavior where checkbox presence indicates checked state
    - In HTML forms, checkboxes send a value when checked, nothing when unchecked

    **Examples**:
        parse_boolean("on")         # True  - standard HTML checkbox value
        parse_boolean("true")       # True  - explicit true
        parse_boolean("1")          # True  - numeric true
        parse_boolean("enabled")    # True  - any non-empty string
        parse_boolean("disabled")   # True  - INTENTIONAL: any string = checkbox was checked!
        parse_boolean("custom")     # True  - custom checkbox value

        parse_boolean("false")      # False - explicit false
        parse_boolean("off")        # False - explicit false
        parse_boolean("0")          # False - explicit false
        parse_boolean("")           # False - empty string = unchecked
        parse_boolean(None)         # False - missing = unchecked

    **Why "disabled" returns True**:
    This is NOT a bug! If a checkbox sends the value "disabled", it means the checkbox
    was checked (present in form data). The actual string content doesn't matter for
    HTML checkboxes - only presence vs absence matters.

    Args:
        value: Value to convert to boolean. Accepts strings, booleans, or None.

    Returns:
        bool: True for truthy values (any non-empty string except explicit false);
              False for falsy values ('off', 'false', '0', '', 'no', False, None)

    Note:
        This function implements HTML form semantics, NOT generic boolean parsing.
        See tests/settings/test_boolean_parsing.py for comprehensive test coverage.
    """
    # Constants for boolean value parsing
    FALSY_VALUES = ("off", "false", "0", "", "no")

    # Handle already-boolean values
    if isinstance(value, bool):
        return value

    # Handle None (missing values)
    if value is None:
        return False

    # Handle string values
    if isinstance(value, str):
        value_lower = value.lower().strip()
        # Explicitly falsy values (empty string, false-like values)
        if value_lower in FALSY_VALUES:
            return False
        # Any other non-empty string = True (HTML checkbox semantics)
        return True

    # For other types (numbers, lists, etc.), use Python's bool conversion
    return bool(value)


def _parse_number(x):
    """Parse number, returning int if it's a whole number, otherwise float."""
    if isinstance(x, bool):
        # Fixed validation text; no configured value is included.
        # bearer:disable python_lang_exception
        raise ValueError("Expected a number")
    f = float(x)
    if not math.isfinite(f):
        # Fixed validation text; no configured value is included.
        # bearer:disable python_lang_exception
        raise ValueError("Expected a finite number")
    if f.is_integer():
        return int(f)
    return f


def _parse_json_value(x):
    """Parse JSON ui_element values.

    DB values (via SQLAlchemy JSON column) arrive as Python objects already.
    Form POST and env var overrides arrive as raw strings and need parsing.
    For example, a textarea containing ``["general"]`` arrives as the string
    ``'[\\r\\n  "general"\\r\\n]'`` which must be decoded into a list.
    """
    if isinstance(x, str):
        stripped = x.strip()
        if stripped:
            try:
                return json.loads(stripped)
            except (json.JSONDecodeError, ValueError, RecursionError):
                # The JSON database column may already contain a decoded
                # scalar string. Preserve that value; environment input is
                # known to be encoded and is parsed strictly by the reader.
                return x
    return x


def _parse_multiselect(x):
    """Parse multiselect value, handling both lists and strings.

    DB values (via SQLAlchemy JSON column) arrive as Python lists already.
    Env var overrides arrive as strings and need parsing — either as JSON
    arrays (e.g. '["markdown","latex"]') or comma-separated values
    (e.g. 'markdown,latex').
    """
    if isinstance(x, list):
        return x
    if isinstance(x, str):
        stripped = x.strip()
        if stripped.startswith(("[", "{")):
            parsed = json.loads(stripped)
            if not isinstance(parsed, list):
                # Fixed validation text; the parsed object is not included.
                # bearer:disable python_lang_exception
                raise ValueError("Expected a list")
            return parsed
        # Comma-separated values are an explicitly supported input format.
        return [item.strip() for item in stripped.split(",") if item.strip()]
    if x is None:
        return None
    # Fixed validation text; the rejected carrier is not included.
    # bearer:disable python_lang_exception
    raise ValueError("Expected a list or comma-separated values")


def _filter_setting_columns(data: dict) -> dict:
    """Filter a dict to only keys that are valid Setting model columns.

    Prevents crashes when default_settings.json contains keys not present
    as columns on the Setting model (e.g. future flags).
    """
    valid_columns = {c.name for c in Setting.__table__.columns}
    return {k: v for k, v in data.items() if k in valid_columns}


_SETTING_METADATA_FIELDS = (
    "value",
    "type",
    "name",
    "description",
    "category",
    "ui_element",
    "options",
    "min_value",
    "max_value",
    "step",
    "visible",
    "editable",
)


# Select settings whose options lists are populated dynamically (at code or
# runtime) rather than from the static defaults JSON. Their static options
# are only a sample of what is legal, so options validation must be skipped
# for them. Re-exported by `web/services/settings_service.py` (single
# source of truth) and imported by tests that assert identity across
# import orders.
DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]


# Re-exported by `api/settings_utils.py` (single source of truth) and
# imported by tests that assert identity across import orders.
def validate_imported_setting_value(
    key: str, value: Any, default_meta: Dict[str, Any]
) -> Optional[str]:
    """Validate a file-supplied setting value against the current-defaults
    schema for its key.

    Mirrors the runtime checks `web.services.settings_service
    .validate_setting` applies on the settings save path (type coercion via
    ``get_typed_setting_value``, options / min / max constraints), but reads
    constraints from the current defaults metadata instead of a stored
    ``Setting`` row. Used by ``import_settings`` so a pre-upgrade export
    cannot resurrect values that are invalid under the current schema
    (#5589).

    Trusted-vs-untrusted rationale (the call-site comments in both
    managers' ``import_settings`` point here): file-supplied values are
    new input, so ``import_settings`` runs this check on them (keys with
    no defaults metadata, and in-memory entries without a ``value``
    field, are stored unchecked). Retained database or in-memory values
    — kept through ``overwrite=False`` or
    ``preserve_environment_locked=True`` to preserve existing user data
    — are not new input and deliberately skip this check. Re-validating
    them would not discard them: on a reject, both managers keep the
    key's existing row or entry, as they do for a rejected file value.
    It would cost the key's metadata refresh instead: where retained
    keys get refreshed metadata (the database manager, and env-locked
    keys in the in-memory one), a key whose stored value fails the
    current rules would keep its previous metadata (``ui_element``,
    options, min/max). "Trusted" in the call-site comments refers to this
    preservation decision; it does not imply prior schema validation.
    Programmatic setters can bypass this check, and older values may
    not satisfy the current defaults schema.

    Returns ``None`` when the value is acceptable, or a human-readable
    reason string when it is not.
    """
    if key == "search.fetch.summary_input_max_length":
        try:
            to_positive_int(value)
        except ValueError:
            return "Value must be a positive integer"

    ui_element = default_meta.get("ui_element")
    if (
        not isinstance(ui_element, str)
        or ui_element not in UI_ELEMENT_TO_SETTING_TYPE
    ):
        return "Unsupported setting type"

    if value == default_meta.get("value"):
        # The value equals the currently shipped default, so the defaults
        # file itself vouches for it. Without this guard, a shipped default
        # that is technically outside its own constraint metadata (e.g. a
        # null optional numeric default) would be rejected and the key
        # dropped on fresh-install seeding.
        #
        # ``app.theme`` used to be the example here: it shipped as ``dark``
        # while the theme registry, which replaces that setting's options at
        # runtime, had no ``dark`` entry. This guard is what let that invalid
        # default seed successfully — and therefore what kept it invisible
        # until the no-JS Save path validated it and reported the user's
        # untouched theme as "failing". The current default, ``hashed``,
        # is the registry ID of the existing Hashed dark palette, selected
        # while light themes still need refinement.
        return None

    try:
        typed_value = get_typed_setting_value(
            key=key,
            value=value,
            ui_element=ui_element,
            default=None,
            check_env=False,
        )
    except SettingsValueError as error:
        return error.reason
    if typed_value is None:
        # A shipped optional default may legitimately be null; the equality
        # guard above accepts that exact value. Otherwise this is either an
        # unconvertible input or a non-default null. The save path rejects it
        # for typed controls, so imports must do the same rather than storing
        # a value that the UI could not save.
        if ui_element == "checkbox":
            return "Value must be a boolean"
        if ui_element in ("number", "slider", "range"):
            return "Value must be a number"
        return None

    if ui_element == "checkbox":
        if not isinstance(typed_value, bool):
            return "Value must be a boolean"
    elif ui_element in ("number", "slider", "range"):
        if not isinstance(typed_value, (int, float)):
            return "Value must be a number"
        min_value = default_meta.get("min_value")
        max_value = default_meta.get("max_value")
        if min_value is not None and typed_value < min_value:
            return f"Value must be at least {min_value}"
        if max_value is not None and typed_value > max_value:
            return f"Value must be at most {max_value}"
    elif ui_element == "select":
        options = default_meta.get("options")
        # Skip options validation for dynamically populated dropdowns —
        # their static options are only a sample of legal values.
        if options and key not in DYNAMIC_SETTINGS:
            allowed_values = [
                opt.get("value") if isinstance(opt, dict) else opt
                for opt in list(options)
            ]
            if typed_value not in allowed_values:
                return "Value must be one of: " + ", ".join(
                    str(v) for v in allowed_values
                )

    return None


_POLICY_AUDIT_KEYS = frozenset(
    {
        "llm.require_local_endpoint",
        "llm.allowed_local_hostnames",
        "embeddings.require_local",
    }
)


def _is_policy_setting(key: str) -> bool:
    """Return True for security-relevant setting keys that need an
    audit-log entry on change. Scope is intentionally narrow so this
    audit hook doesn't widen into general settings-change logging.

    Narrow is not the same as enumerated: the ``policy.`` arm is a PREFIX
    match, not an allowlist, so this covers every current and future
    ``policy.*`` key including ones a caller creates. See
    ``_policy_audit_value`` for why the audit line therefore cannot assume
    the values it records are scalars from a fixed vocabulary.
    """
    if key.startswith("policy."):
        return True
    return key in _POLICY_AUDIT_KEYS


def _policy_audit_allowed_values(
    default_meta: Optional[Dict[str, Any]],
) -> Optional[List[Any]]:
    """The fixed vocabulary a policy setting's value must come from, or
    ``None`` when the key has no such vocabulary.

    Only a ``select``/``radio`` row with a static ``options`` list has one.
    Used by ``_policy_audit_value`` to decide whether a STRING value is
    safe to record verbatim: a value drawn from a shipped option list is a
    known token (``adaptive``, ``strict``, ...), not free-form input, so it
    cannot be a credential.
    """
    if not isinstance(default_meta, dict):
        return None
    if default_meta.get("ui_element") not in ("select", "radio"):
        return None
    options = default_meta.get("options")
    if not options:
        return None
    return [
        opt.get("value") if isinstance(opt, dict) else opt
        for opt in list(options)
    ]


def _policy_audit_value(
    value: Any, allowed_values: Optional[List[Any]] = None
) -> str:
    """Render a value for the ``policy.*`` audit line so the line itself
    can never become a credential-disclosure channel.

    Recording old/new is the whole point of an audit entry, so this does
    not drop the information -- it drops the part that can carry a secret.
    The audit hook's scope was previously described as "``policy.*`` plus
    three keys holding booleans and a hostname list", which is not what the
    code does: ``_is_policy_setting`` PREFIX-matches ``policy.``, and three
    keys already in scope ship as free-form, editable JSON with no item
    schema at all -- ``policy.trusted_search_engines``,
    ``policy.trusted_inference_providers`` and
    ``llm.allowed_local_hostnames`` are all ``ui_element="json"``,
    ``editable=True``, default ``[]``. Their egress validator
    (``validate_trusted_search_engines``) ``continue``s past any non-``str``
    entry rather than rejecting it, so a list like
    ``[{"api_key": "sk-...", "name": "x"}]`` is accepted and reaches
    ``set_setting`` -- which then wrote both ``old=`` and ``new=`` to the
    log in plaintext at WARNING. That line also sits OUTSIDE the
    ``if commit:`` block, so the bulk save routes (``commit=False``) reach
    it too; it is not a PUT-only path (#6201 round 7).

    So plaintext is kept ONLY where the value is PROVEN unable to carry
    free-form text, and the proof is a check rather than a comment:

      * ``None``/``bool``/``int``/``float`` -- no string content at all;
      * a ``str`` that is one of the key's own registered ``select``
        options (see ``_policy_audit_allowed_values``) -- a fixed
        vocabulary shipped in ``defaults/``, e.g. ``policy.egress_scope``.

    Everything else -- a free-form string, a dict, a list, any other
    type -- is recorded as STRUCTURAL metadata only: the type name, plus
    the length when it is a string or the entry count when it is a
    container. No token derived from the value's CONTENT appears in the
    rendering. An earlier version logged a truncated SHA-256 of the
    value's canonical JSON; a stable unkeyed digest -- even cut to 64
    bits, re-truncated, or keyed with a key shipped in the repository --
    is an offline dictionary oracle: anyone holding the log can confirm
    a guessed credential by hashing candidates offline, which is exactly
    the disclosure this rendering exists to prevent (#6201 round 8: a
    common password was recovered from the logged prefix by an offline
    candidate list).

    The cost of structure-only recording is honest and deliberate:
    change detection on free-form values is COARSE. "still an
    11-character string" / "the trusted-engine list went from 2 entries
    to 3" is all an auditor gets; two different 11-character secrets
    render IDENTICALLY, so the line can never confirm which candidate is
    the real value. A string's length narrows a candidate list but
    confirms nothing (every same-length candidate matches), which is
    what separates it from a digest. Scalars from a fixed vocabulary and
    values with no string content stay verbatim, so the policy decisions
    themselves -- the actual subject of this audit -- remain fully
    observable.
    """
    if value is None or isinstance(value, (bool, int, float)):
        return repr(value)
    if allowed_values and value in allowed_values:
        return repr(value)
    if isinstance(value, (dict, list)):
        return f"<{type(value).__name__}[{len(value)}]>"
    if isinstance(value, str):
        return f"<str[{len(value)}]>"
    return f"<{type(value).__name__}>"


def is_valid_setting_key(key: Any) -> bool:
    """Return True if *key* is a well-formed setting key.

    A valid key is a non-empty string of dot-separated segments where each
    segment is non-empty and contains no whitespace or ASCII control characters.
    This rejects the
    malformed keys that corrupt prefix/namespace lookups (see #4840) — a
    trailing dot ``"foo."``, a leading dot ``".foo"``, an empty segment
    ``"foo..bar"``, a blank key — as well as keys carrying whitespace
    (``" foo"``, ``"foo. bar"``, ``"llm.o k"``), which would also break the
    ``key.split(".")`` → ``LDR_...`` env-var mapping. A malformed row makes
    ``get_setting("foo")`` return a ``{"": value}`` wrapper dict, which the
    UI renders as ``[object Object]``.

    Used on both sides: writes reject new malformed keys, and both read paths
    (``SettingsManager.get_setting`` on the DB and ``get_setting_from_snapshot``
    on a settings snapshot) filter malformed rows out of a prefix read so a
    stray legacy row can never wrap a leaf value into an ``[object Object]``
    dict.
    """
    if not isinstance(key, str) or not key:
        return False
    return all(
        seg
        and not any(c.isspace() or ord(c) < 0x20 or ord(c) == 0x7F for c in seg)
        for seg in key.split(".")
    )


def _infer_ui_element(value: Any, current: str = "text") -> str:
    """Infer the appropriate ui_element string from a Python value's type.

    Args:
        value: The value to infer the ui_element from.
        current: The existing ui_element. If it is already something more
            specific than ``"text"``, it is kept as-is.
    """
    if current != "text":
        return current
    if isinstance(value, bool):
        return "checkbox"
    if isinstance(value, (int, float)):
        return "number"
    if isinstance(value, (list, dict)):
        return "json"
    return "text"


# Default categories for each typed setting prefix, used by the self-heal
# block in set_setting() when a row's type column doesn't match its key
# prefix. The values mirror the canonical strings in
# web/routers/settings.py: a legacy chat.* row with type=APP and a
# stale category gets repointed to type=CHAT + category="chat" on next
# save. We use the most-general category per prefix here — sub-classifying
# llm_general vs llm_parameters depends on the specific key, but the
# self-heal only fires when the row was already mis-typed, so over-
# generalizing the category is preferable to leaving it stale.
_INFERRED_CATEGORY: Dict[str, str] = {
    "llm.": "llm_general",
    "search.": "search_general",
    "report.": "report_parameters",
    "database.": "database_parameters",
    "chat.": "chat",
}


UI_ELEMENT_TO_SETTING_TYPE: Dict[str, Callable[..., Any]] = {
    "text": str,
    "json": _parse_json_value,
    "password": str,
    "select": str,
    "number": _parse_number,
    "range": _parse_number,  # Same behavior as number for consistency
    "checkbox": parse_boolean,
    "textarea": str,
    "multiselect": _parse_multiselect,
}


def _reject_json_constant(_value: str):
    """Reject JSON's non-standard NaN/Infinity extensions without echoing input."""
    # The rejected argument is unused; this diagnostic is constant.
    # bearer:disable python_lang_exception
    raise ValueError("Expected a finite JSON number")


def _parse_environment_boolean(value: str) -> bool:
    """Environment input has no HTML checkbox presence semantics."""
    normalized = value.strip().lower()
    if normalized in ("true", "1", "yes", "on", "enabled"):
        return True
    if normalized in ("false", "0", "no", "off", "disabled"):
        return False
    # Fixed validation text; the environment value is not included.
    # bearer:disable python_lang_exception
    raise ValueError("Expected a boolean token")


def get_typed_setting_value(
    key: str,
    value: Any,
    ui_element: str,
    default: Any = None,
    check_env: bool = True,
) -> Any:
    """
    Extracts the value for a particular setting, ensuring that it has the
    correct type.

    Args:
        key: The setting key.
        value: The setting value from the database.
        ui_element: The setting UI element ID.
        default: Default value to return when the setting is unset (None).
        check_env: If true, it will check the environment variable for
            this setting before reading from the DB.

    Returns:
        The typed value. The separate fetch-summary budget preserves an
        invalid value, with a warning, until a summary fetch rejects it.

    Raises:
        SettingsValueError: The selected value cannot be converted, or its
            declared setting type is unsupported. Invalid overrides never
            fall through to a lower-priority value.

    """
    return _convert_setting_value(
        key,
        value,
        ui_element,
        default,
        check_env_setting(key) if check_env else None,
    )


def _convert_setting_value(
    key: str,
    value: Any,
    ui_element: str,
    default: Any,
    env_value: str | None,
) -> Any:
    """Convert a selected value using an already resolved environment source."""
    source = "environment" if env_value is not None else "value"
    if (
        not isinstance(ui_element, str)
        or ui_element not in UI_ELEMENT_TO_SETTING_TYPE
    ):
        # Report only the setting key, source and this fixed reason.
        # bearer:disable python_lang_exception
        raise SettingsValueError(
            key, source=source, reason="unsupported setting type"
        )
    setting_type = UI_ELEMENT_TO_SETTING_TYPE[ui_element]
    if env_value is not None:
        value = env_value
    if key == "search.fetch.summary_input_max_length":
        # This exact positive integer is validated again before a summary
        # fetch. Keep a malformed budget visible without blocking unrelated
        # research modes or the settings page used to repair it. All other
        # settings and unsupported UI types retain the strict read contract.
        try:
            return to_positive_int(value)
        except ValueError:
            pass
        location = (
            f"LDR_{'_'.join(key.split('.')).upper()}"
            if env_value is not None
            else f"Setting {key}"
        )
        logger.warning(
            "{} must be a positive integer (value omitted). Summary-mode "
            "fetches will fail until it is corrected.",
            location,
        )
        return value

    if value is None:
        return default
    try:
        if ui_element == "json" and env_value is not None:
            return json.loads(value, parse_constant=_reject_json_constant)
        if ui_element == "checkbox" and env_value is not None:
            # Environment variables are configuration, not submitted HTML
            # checkboxes. An unrecognized token must not enable a feature.
            return _parse_environment_boolean(value)
        return setting_type(value)
    except (ValueError, TypeError, OverflowError, RecursionError):
        pass
    # Raise outside the handler: even a suppressed __context__ can contain
    # credentials from float()/json.loads() errors. See #6201.
    # The control type is allowlisted above; no value or parser error is used.
    # bearer:disable python_lang_exception
    raise SettingsValueError(
        key, source=source, reason=f"expected a valid {ui_element} value"
    )


def check_env_setting(key: str) -> str | None:
    """
    Checks environment variables for a particular setting.

    Args:
        key: The database key for the setting.

    Returns:
        The setting from the environment variables, or None if the variable
        is not set or is empty.

    Note:
        Empty environment variables ("") are treated as unset. This is standard
        practice across the ecosystem — see CPython's official docs (PYTHON*
        env vars require "a non-empty string"), botocore PR #1687, Pallets/Click
        PR #2223, and Vercel Turborepo PR #6929. Orchestration tools like Unraid,
        Terraform, and Kubernetes manifests often cannot conditionally omit env
        var declarations, so they pass "" for unconfigured values. Treating ""
        as unset prevents these empty strings from overriding database defaults.
        See: https://github.com/LearningCircuit/local-deep-research/pull/3362

    """
    env_variable_name = f"LDR_{'_'.join(key.split('.')).upper()}"
    env_value = os.getenv(env_variable_name)
    # Treat empty string as unset — orchestration tools (Unraid, Terraform, K8s)
    # often cannot omit env var declarations and pass "" for unconfigured values.
    if env_value is not None and env_value != "":
        logger.debug(f"Overriding {key} setting from environment variable.")
        return env_value
    if env_value == "":
        logger.warning(
            "Environment variable {} is set but empty — "
            "ignoring it and falling back to DB/default for setting '{}'. "
            "This is expected on Unraid or Docker templates that create "
            "all variables even when left blank. To suppress this warning, "
            "remove the variable from your environment or set a value.",
            env_variable_name,
            key,
        )
    return None


def apply_environment_overrides_to_snapshot(
    snapshot: Dict[str, Any],
) -> Dict[str, Any]:
    """Return a copied snapshot with current LDR_* overrides reapplied.

    Queued research snapshots can outlive a process restart. Environment values
    are operator policy and retain highest precedence at actual dispatch time.
    """
    if not isinstance(snapshot, dict):
        return snapshot
    effective = dict(snapshot)
    # Old or partial queued snapshots may omit keys added after enqueue. Use
    # the registered defaults as metadata so every active LDR_* override is
    # injected at dispatch, not only overrides for keys already serialized.
    default_metadata = SettingsManager().default_settings
    candidate_keys = set(snapshot) | set(default_metadata)
    # Function-local import: the binding module imports this one.
    from ..config.openai_endpoint_binding import (
        OPENAI_ENDPOINT_API_KEY_SETTING,
        OPENAI_ENDPOINT_KEY_OWNER_MARKER,
    )

    if OPENAI_ENDPOINT_KEY_OWNER_MARKER in snapshot:
        # Preserve the value associated with the recorded owner. Envelope
        # reads already apply the current environment key at dispatch; it
        # must not replace the configured value retained in the envelope.
        # Raw entries are explicit or cleared values and stay unchanged.
        candidate_keys.discard(OPENAI_ENDPOINT_API_KEY_SETTING)
    for key in candidate_keys:
        if check_env_setting(str(key)) is None:
            continue
        stored = snapshot.get(key, default_metadata.get(key))
        metadata = stored if isinstance(stored, dict) else None
        current = metadata.get("value") if metadata is not None else stored
        if metadata is not None:
            ui_element = str(metadata.get("ui_element", "text"))
        elif isinstance(current, bool):
            ui_element = "checkbox"
        elif isinstance(current, (list, dict)):
            ui_element = "json"
        elif isinstance(current, (int, float)):
            ui_element = "number"
        else:
            ui_element = "text"
        typed = get_typed_setting_value(
            str(key), current, ui_element, default=current, check_env=True
        )
        if metadata is not None:
            updated = dict(metadata)
            updated["value"] = typed
            updated["editable"] = False
            effective[key] = updated
        else:
            effective[key] = typed
    return effective


class SettingsManager(ISettingsManager):
    """
    Manager for handling application settings with database storage and file fallback.
    Provides methods to get and set settings, with the ability to override settings in memory.
    """

    def __init__(
        self,
        db_session: Optional[Session] = None,
        owns_session: bool = False,
        *,
        seed_defaults: bool = True,
    ):
        """
        Initialize the settings manager

        Args:
            db_session: SQLAlchemy session for database operations
            owns_session: If True, close() will close the session.
                Defaults to False (safe for borrowed sessions).  Set to True
                only when this manager created/owns the session — currently
                only get_settings_manager() in db_utils.py does this.
            seed_defaults: If True (the default), an empty settings table is
                seeded from the defaults files and COMMITTED on
                ``db_session`` here. Pass False for a read-only manager on a
                session that belongs to someone else's transaction (e.g. a
                deletion reading the user's snapshot mid-transaction): reads
                still merge the in-memory defaults, so an empty table reads
                as all-defaults without writing anything.
        """
        self.db_session = db_session
        self._owns_session = owns_session
        self._closed = False
        self.db_first = True  # Always prioritize DB settings

        # Store the thread ID this instance was created in
        self._creation_thread_id = threading.get_ident()

        # Initialize settings lock as None - will be checked lazily
        self.__settings_locked: Optional[bool] = None

        # Auto-initialize settings if database is empty
        if self.db_session and seed_defaults:
            self._ensure_settings_initialized()

    def close(self):
        """Close the DB session if this manager owns it.

        Borrowed sessions (owns_session=False) are left open for their
        owner to close (e.g. a caller that passed in its own db_session).
        Safe to call multiple times — subsequent calls are no-ops.
        """
        if self._owns_session and self.db_session is not None:
            try:
                logger.debug("Closing owned DB session in SettingsManager")
                self.db_session.close()
            except Exception:
                logger.warning(
                    "Failed to close SettingsManager DB session — "
                    "connection may leak",
                )
        self._closed = True
        self.db_session = None

    def _ensure_settings_initialized(self):
        """Ensure settings are initialized in the database."""
        # Check if we have any settings at all
        from ..database.models import Setting

        if self.db_session is None:
            # Fixed lifecycle text; no session contents are included.
            # bearer:disable python_lang_exception
            raise RuntimeError("Database session is not initialized")
        settings_count = self.db_session.query(Setting).count()

        if settings_count == 0:
            logger.info("No settings found in database, loading defaults")
            # Trusted first-run seeding bypasses the lock via import_settings
            # directly — the load_from_defaults_file wrapper no longer
            # accepts override_locked (#5841). Time it directly too: routing
            # through the wrapper used to log this for free.
            start = time.perf_counter()
            row_count = len(self.default_settings)
            self.import_settings(
                self.default_settings, commit=True, override_locked=True
            )
            elapsed_ms = (time.perf_counter() - start) * 1000
            logger.info(
                f"Default settings loaded successfully: {row_count} "
                f"settings in {elapsed_ms:.0f}ms"
            )

    def _check_thread_safety(self):
        """Check if this instance is being used in the same thread it was created in."""
        current_thread_id = threading.get_ident()
        if self.db_session and current_thread_id != self._creation_thread_id:
            # Only numeric IDs from threading.get_ident() are interpolated.
            # bearer:disable python_lang_exception
            raise RuntimeError(
                f"SettingsManager instance created in thread {self._creation_thread_id} "
                f"is being used in thread {current_thread_id}. This is not thread-safe! "
                f"Create a new SettingsManager instance within the current thread context."
            )

    @property
    def settings_locked(self) -> bool:
        """Check if settings are locked (lazy evaluation)."""
        if self.__settings_locked is None:
            try:
                self.__settings_locked = parse_boolean(
                    self.get_setting("app.lock_settings", False)
                )
                if self.__settings_locked:
                    logger.info(
                        "Settings are locked. Disabling all settings changes."
                    )
            except SettingsValueError:
                raise
            except Exception:
                logger.bind(policy_audit=True).warning(
                    "Failed to check settings lock status; denying changes"
                )
                self.__settings_locked = True
        return bool(self.__settings_locked)

    def _is_environment_locked(self, key: str, operation: str) -> bool:
        if check_env_setting(key) is None:
            return False
        logger.bind(policy_audit=True).warning(
            "environment-locked setting rejected at {}", operation, key=key
        )
        return True

    @functools.cached_property
    def default_settings(self) -> Dict[str, Any]:
        """
        Returns:
            The default settings, loaded from JSON files and merged.
            Automatically discovers and loads all .json files in the defaults
            directory and its subdirectories.
            Theme options are dynamically injected from the theme registry.

        """
        settings: Dict[str, Any] = {}

        try:
            # Get the defaults package path
            defaults_path = Path(defaults.__file__).parent

            # Find all JSON files recursively in the defaults directory
            # nosemgrep: semgrep.rules.path-traversal-risk, reason: Only JSON defaults beneath the installed defaults package are enumerated
            json_files = sorted(defaults_path.rglob("*.json"))

            logger.debug(f"Found {len(json_files)} JSON settings files")

            # Load and merge all JSON files
            for json_file in json_files:
                try:
                    # nosemgrep: semgrep.rules.path-traversal-risk, reason: Each file comes from the installed defaults package JSON glob above
                    with open(json_file, "r", encoding="utf-8-sig") as f:
                        file_settings = json.load(f)

                    # Get relative path for logging
                    relative_path = json_file.relative_to(defaults_path)

                    # Warn about key conflicts
                    conflicts = set(settings.keys()) & set(file_settings.keys())
                    if conflicts:
                        logger.warning(
                            f"Keys {conflicts} from {relative_path} "
                            f"override existing values"
                        )

                    settings.update(file_settings)
                    logger.debug(f"Loaded {relative_path}")

                except json.JSONDecodeError:
                    logger.exception(f"Invalid JSON in {json_file}")
                except Exception:
                    logger.warning(f"Could not load {json_file}")

        except Exception:
            logger.warning("Error loading settings files")

        # Inject dynamic theme options from theme registry
        if "app.theme" in settings:
            try:
                from local_deep_research.web.themes import theme_registry

                settings["app.theme"]["options"] = (
                    theme_registry.get_settings_options()
                )
            except ImportError:
                # Theme registry not available, use static options from JSON
                pass

        # Inject search strategy options from code (single source of truth)
        if "search.search_strategy" in settings:
            from local_deep_research.constants import get_available_strategies

            strategies = get_available_strategies()
            settings["search.search_strategy"]["options"] = [
                {"label": s["label"], "value": s["name"]} for s in strategies
            ]

        logger.debug(f"Loaded {len(settings)} total settings")
        return settings

    def __get_typed_setting_value(
        self,
        setting: Setting,
        default: Any = None,
        check_env: bool = True,
    ) -> Any:
        """
        Extracts the value for a particular setting, ensuring that it has the
        correct type.

        Args:
            setting: The setting to get the value for.
            default: Default value to return if the value of the setting is
                invalid.
            check_env: If true, it will check the environment variable for
                this setting before reading from the DB.

        Returns:
            The value of the setting.

        """
        return get_typed_setting_value(
            str(setting.key),
            setting.value,
            str(setting.ui_element),
            default=default,
            check_env=check_env,
        )

    def __query_settings(self, key: str | None = None) -> List[Setting]:
        """
        Abstraction for querying settings that also transparently handles
        reading the default settings file if the DB is not enabled.

        Args:
            key: The key to read. If None, it will read everything.

        Returns:
            The settings it queried.

        """
        if self.db_session:
            self._check_thread_safety()
            query = self.db_session.query(Setting)
            if key is not None:
                # This will find exact matches and any subkeys.
                #
                # The ``startswith`` arm is intentionally restricted to
                # subkeys that have AT LEAST one character past the dot
                # (``key LIKE 'foo.%' AND key != 'foo.'``). The bare
                # ``LIKE 'foo.%'`` form silently also matches ``'foo.'``
                # (because ``%`` matches zero or more chars), which would
                # treat a malformed trailing-dot duplicate row as a
                # subkey of ``foo`` and cause ``get_setting`` to return
                # a 2-key dict (``{"": value, "foo": value}``) instead
                # of the primitive. See embedding-settings page bug.
                # escape_like: the requested key is user-supplied (e.g. via
                # /settings/api/bulk?keys[]=...), so escape its LIKE wildcards
                # or ``keys[]=%`` would match every dotted key and dump the
                # whole table. The ``!= f"{key}."`` guard compares against the
                # raw stored key, so it stays unescaped.
                query = query.filter(
                    or_(
                        Setting.key == key,
                        and_(
                            Setting.key.like(
                                f"{escape_like(key)}.%", escape="\\"
                            ),
                            Setting.key != f"{key}.",
                        ),
                    )
                )
            return query.all()

        logger.debug(
            "DB is disabled, reading setting '{}' from defaults file.", key
        )

        settings = []
        for candidate_key, setting in self.default_settings.items():
            if key is None or (
                candidate_key == key or candidate_key.startswith(f"{key}.")
            ):
                settings.append(
                    Setting(
                        key=candidate_key,  # gitleaks:allow
                        **_filter_setting_columns(setting),
                    )
                )

        return settings

    def get_setting(
        self, key: str, default: Any = None, check_env: bool = True
    ) -> Any:
        """
        Get a setting value

        Args:
            key: Setting key
            default: Default value if setting is not found
            check_env: If true, it will check the environment variable for
                this setting before reading from the DB.

        Returns:
            Setting value or default if not found. An environment override
            for a key with no row and no declared type is returned as the
            raw string.

        Raises:
            SettingsValueError: The selected value cannot be converted to its
                declared type, or that type is unsupported.
        """
        if self._closed:
            logger.error(
                "SettingsManager.get_setting('{}') called after close() — "
                "this is a bug; the caller should not reuse a closed manager",
                key,
            )
            # Fixed lifecycle text; no setting value is included.
            # bearer:disable python_lang_exception
            raise RuntimeError(
                "SettingsManager has been closed. "
                "Create a new instance or call close() only at end of lifecycle."
            )

        # First check if this is an env-only setting
        if env_registry.is_env_only(key):
            return env_registry.get_strict(key, default, check_env=check_env)

        # If using database first approach and session available, check database
        try:
            settings = self.__query_settings(key)
            # Drop malformed subkey rows (e.g. legacy "foo." / "foo.." keys)
            # that slip past the SQL prefix filter. #4852 excludes only the
            # exact single-trailing-dot row; a "foo.." / "foo. " row would
            # otherwise turn a leaf read into a `{".": v}`-style wrapper dict
            # rendered as `[object Object]` (#4840). The exact-match row is
            # always kept so a direct read of a malformed key still works.
            #
            # An operator-only key is never read as a namespace. Its child
            # rows, whether saved before the policy or matched by LIKE as an
            # ASCII case variant, are dropped, so the read returns the exact
            # row's value, the LDR_* override or the default, never a
            # mapping.
            exact_row_only = is_operator_only_setting(key)
            settings = [
                s
                for s in settings
                if str(s.key) == key
                or (not exact_row_only and is_valid_setting_key(str(s.key)))
            ]
            if len(settings) == 1 and str(settings[0].key) == key:
                # Only an exact row is a bottom-level key. A lone child row
                # still represents a namespace and must return a mapping,
                # matching get_setting_from_snapshot().
                return self.__get_typed_setting_value(
                    settings[0], default, check_env
                )
            if settings:
                # This is a higher-level key.
                settings_map = {}
                for setting in settings:
                    output_key = str(setting.key).removeprefix(f"{key}.")
                    settings_map[output_key] = self.__get_typed_setting_value(
                        setting, default, check_env
                    )
                return settings_map
        except SQLAlchemyError:
            logger.exception(f"Error retrieving setting {key} from database")
            if key == "app.lock_settings":
                raise

        # Check env var before returning default (setting not in DB)
        if check_env:
            env_value = check_env_setting(key)
            if env_value is not None:
                default_meta = self.default_settings.get(key)
                if default_meta and isinstance(default_meta, dict):
                    ui_element = default_meta.get("ui_element", "text")
                    return get_typed_setting_value(
                        key,
                        None,
                        ui_element,
                        default=default,
                        check_env=True,
                    )
                # No type is declared, so there is nothing to validate the
                # override against: return it unconverted, as before strict
                # reads. Untyped keys such as app.api_rate_limit and
                # app.enable_api are read on every /api/v1 request, and their
                # consumers parse the string themselves. Typed keys still
                # raise above on a value that cannot be converted.
                logger.warning(
                    "Setting '{}' has env var override but is not in "
                    "defaults — returning raw string without type "
                    "conversion. Add this setting to a defaults JSON "
                    "file with a ui_element type to enable proper "
                    "type conversion.",
                    key,
                )
                return env_value

        # Return default if not found
        return default

    def get_bool_setting(
        self, key: str, default: bool = False, check_env: bool = True
    ) -> bool:
        """
        Get a setting value as a boolean, handling string conversion.

        Args:
            key: Setting key
            default: Default boolean value if setting is not found
            check_env: If true, it will check the environment variable for
                this setting before reading from the DB.

        Returns:
            Boolean value of the setting
        """
        value = self.get_setting(key, default, check_env)
        return to_bool(value, default)

    def set_setting(self, key: str, value: Any, commit: bool = True) -> bool:
        """
        Set a setting value

        Args:
            key: Setting key
            value: Setting value
            commit: Whether to commit the change

        Returns:
            True if successful, False otherwise
        """
        if self._closed:
            logger.error(
                "SettingsManager.set_setting('{}') called after close() — "
                "this is a bug; the caller should not reuse a closed manager",
                key,
            )
            # Fixed lifecycle text; no setting value is included.
            # bearer:disable python_lang_exception
            raise RuntimeError(
                "SettingsManager has been closed. "
                "Create a new instance or call close() only at end of lifecycle."
            )
        if not self.db_session:
            logger.error(
                "Cannot edit setting {} because no DB was provided.", key
            )
            return False
        if is_operator_only_setting(key):
            logger.bind(policy_audit=True).warning(
                "operator-only setting rejected at set_setting", key=key
            )
            return False
        if self.settings_locked:
            logger.error("Cannot edit setting {} because they are locked.", key)
            return False
        if self._is_environment_locked(key, "set_setting"):
            return False

        # Always update database if available
        try:
            self._check_thread_safety()
            setting = (
                self.db_session.query(Setting)
                .filter(Setting.key == key)
                .first()
            )
            # Capture old value for the policy-change audit log below.
            old_value = setting.value if setting is not None else None
            if setting:
                if not setting.editable:
                    logger.error(
                        "Cannot change setting '{}' because it "
                        "is marked as non-editable.",
                        key,
                    )
                    return False

                setting.value = value  # type: ignore[assignment]
                setting.updated_at = (  # type: ignore[assignment]
                    func.now()
                )  # Explicitly set the current timestamp

                # Self-heal stale ui_element from before inference was added
                setting.ui_element = _infer_ui_element(
                    value, setting.ui_element
                )

                # Self-heal stale type from before the prefix dispatch was
                # added (e.g. legacy chat.* rows created with type=APP).
                # Also re-points category to the canonical per-prefix
                # value, since a row with the wrong type column was
                # almost certainly created before category dispatch was
                # in place either.
                inferred_type: Optional[SettingType] = None
                inferred_category: Optional[str] = None
                for prefix, category in _INFERRED_CATEGORY.items():
                    if key.startswith(prefix):
                        if prefix == "llm.":
                            inferred_type = SettingType.LLM
                        elif prefix == "search.":
                            inferred_type = SettingType.SEARCH
                        elif prefix == "report.":
                            inferred_type = SettingType.REPORT
                        elif prefix == "database.":
                            inferred_type = SettingType.DATABASE
                        elif prefix == "chat.":
                            inferred_type = SettingType.CHAT
                        inferred_category = category
                        break
                # Only self-heal when the key matches a known prefix. Keys
                # outside the dispatch map (e.g. focused_iteration.*,
                # langgraph_agent.* which ship as type=SEARCH) must keep their
                # shipped type — defaulting to APP here would wrongly demote
                # them on every edit.
                if inferred_type is not None and setting.type != inferred_type:
                    setting.type = inferred_type  # type: ignore[assignment]
                    if inferred_category is not None:
                        setting.category = inferred_category  # type: ignore[assignment]
            else:
                # Refuse to CREATE a new row for a malformed key (e.g. a
                # trailing-dot key like "foo."). Existing rows are still
                # updatable above; this only blocks minting new corruption.
                if not is_valid_setting_key(key):
                    logger.error(
                        "Refusing to create setting with malformed key {!r}",
                        key,
                    )
                    return False

                # Determine setting type from key
                setting_type = SettingType.APP
                if key.startswith("llm."):
                    setting_type = SettingType.LLM
                elif key.startswith("search."):
                    setting_type = SettingType.SEARCH
                elif key.startswith("report."):
                    setting_type = SettingType.REPORT
                elif key.startswith("database."):
                    setting_type = SettingType.DATABASE
                elif key.startswith("chat."):
                    setting_type = SettingType.CHAT

                # Infer ui_element from the value type
                ui_element = _infer_ui_element(value)

                # Create a new setting
                new_setting = Setting(
                    key=key,
                    value=value,
                    type=setting_type,
                    name=key.split(".")[-1].replace("_", " ").title(),
                    ui_element=ui_element,
                    description=f"Setting for {key}",
                )
                self.db_session.add(new_setting)

            if commit:
                self.db_session.commit()
                # Emit WebSocket event for settings change
                self._emit_settings_changed([key])

            # N16: audit log on policy.* / llm.require_local_endpoint /
            # embeddings.require_local changes. Targeted scope — only
            # security-relevant settings are logged, to avoid widening
            # this PR into a general audit-log refactor.
            #
            # The values are rendered through ``_policy_audit_value``, NOT
            # interpolated raw: ``policy.`` is a prefix match and three of
            # the keys in scope hold free-form, user-editable JSON, so a
            # raw ``old={}``/``new={}`` here is a plaintext credential sink
            # (#6201 round 7). This line is also outside the ``if commit:``
            # block above on purpose — the bulk save routes pass
            # ``commit=False`` and must still be audited.
            if _is_policy_setting(key):
                try:
                    _default_meta = self.default_settings.get(key)
                except Exception:
                    # An audit line must never be able to fail the write it
                    # is auditing; without the metadata every string simply
                    # falls back to the structural form.
                    _default_meta = None
                _audit_options = _policy_audit_allowed_values(_default_meta)
                logger.bind(policy_audit=True).warning(
                    "policy setting changed | key={} old={} new={}",
                    key,
                    _policy_audit_value(old_value, _audit_options),
                    _policy_audit_value(value, _audit_options),
                )

            return True
        except SQLAlchemyError:
            logger.exception(f"Error setting value for key: {key}")
            self.db_session.rollback()
            return False

    def clear_cache(self):
        """Clear the settings cache."""
        self.__dict__.pop("default_settings", None)
        logger.debug("Settings cache cleared")

    def get_all_settings(
        self,
        bypass_cache: bool = False,
        include_environment_overrides: bool = True,
        strict: bool = False,
    ) -> Dict[str, Any]:
        """
        Get all settings, merging defaults with database values.

        This ensures that new settings added to defaults.json automatically
        appear in the UI without requiring a database reset.

        Args:
            bypass_cache: If True, bypass the cache and read directly from database
            include_environment_overrides: If True, validate effective values
                and overlay current LDR_* values. False exports stored values
                and metadata unchanged, including invalid values for repair.
            strict: If True, propagate database and enum query failures.

        Returns:
            Dictionary of all settings

        Raises:
            SettingsValueError: With ``include_environment_overrides``, any
                effective value cannot be converted to its declared type, or
                that type is unsupported.
        """
        if self._closed:
            logger.error(
                "SettingsManager.get_all_settings() called after close() — "
                "this is a bug; the caller should not reuse a closed manager",
            )
            # Fixed lifecycle text; no settings snapshot is included.
            # bearer:disable python_lang_exception
            raise RuntimeError(
                "SettingsManager has been closed. "
                "Create a new instance or call close() only at end of lifecycle."
            )

        result = {}

        # Start with defaults so new settings are always included
        for key, default_setting in self.default_settings.items():
            result[key] = dict(default_setting)

        # Override with database settings
        try:
            db_settings = self.__query_settings() if self.db_session else None
        except (SQLAlchemyError, LookupError):
            # LookupError fires when a row's `type` column holds an enum
            # value that's no longer in `SettingType` (e.g. legacy 'CHAT'-
            # typed rows from removed features). The previous handler only
            # caught SQLAlchemyError, so a single stale row would crash the
            # whole snapshot — and every caller of get_all_settings (the
            # /settings/api endpoint, benchmark start, research start, MCP
            # entry points, …) downstream of it. Falling back to defaults-
            # only is strictly safer than crashing.
            logger.exception(
                "Error querying settings from database in get_all_settings"
            )
            if strict:
                raise
            db_settings = []

        if db_settings is None:
            # Project defaults directly to the same metadata shape as a row.
            # Bulk reads do not need SQLAlchemy instrumentation or temporary
            # Setting objects just to copy their attributes back into dicts.
            selected_settings = (
                (
                    str(key),
                    {
                        field: metadata.get(field)
                        for field in _SETTING_METADATA_FIELDS
                    },
                )
                for key, metadata in self.default_settings.items()
            )
        else:
            selected_settings = (
                (
                    str(setting.key),
                    {
                        "value": setting.value,
                        "type": setting.type,
                        "name": setting.name,
                        "description": setting.description,
                        "category": setting.category,
                        "ui_element": setting.ui_element,
                        "options": setting.options,
                        "min_value": setting.min_value,
                        "max_value": setting.max_value,
                        "step": setting.step,
                        "visible": setting.visible,
                        "editable": setting.editable,
                    },
                )
                for setting in db_settings
            )

        for key, metadata in selected_settings:
            if key.startswith("_"):
                continue  # internal snapshot keys are run metadata, never rows
            # Handle type field - it might be a string or an enum
            setting_type = metadata["type"]
            if hasattr(setting_type, "name"):
                setting_type = setting_type.name

            # Log if this is a custom setting not in defaults
            if key not in result:
                logger.debug(
                    f"Database contains custom setting not in "
                    f"defaults: {key} (type={setting_type}, "
                    f"category={metadata['category']})"
                )

            # Override the default with the full database row — value AND
            # schema (options, description, name, constraints).
            #
            # This is deliberate, not a bug: the DB row is a self-contained
            # snapshot. Schema is NOT read live from JSON on every call; it is
            # reconciled from the JSON defaults only at version bump, via the
            # direct `import_settings(overwrite=False, override_locked=True)`
            # bootstrap calls in `database/initialize.py` and post-login in
            # `web/routers/auth.py`). Every release bumps the package version, so
            # JSON metadata changes reach a user on their next login after
            # upgrade. Do NOT change this to overlay defaults schema on every
            # read — it bypasses that version gate and breaks the snapshot
            # invariant (clean export/import round-trip, no mid-session drift).
            # See closed PR #2474 for the rejected schema/value-separation
            # approach and the reasoning.
            metadata["type"] = setting_type
            if (
                include_environment_overrides and self.settings_locked
            ) or is_operator_only_setting(key):
                metadata["editable"] = False
            result[key] = metadata

        # Validate only after choosing the effective row. A valid database row
        # may replace an obsolete default, and a valid environment override may
        # replace an invalid stored value. Neither may hide an invalid selected
        # value by silently substituting another setting.
        if include_environment_overrides:
            for key, metadata in result.items():
                env_value = check_env_setting(key)
                typed_value = _convert_setting_value(
                    key,
                    metadata.get("value"),
                    metadata.get("ui_element", "text"),
                    None,
                    env_value,
                )
                if env_value is not None:
                    metadata["value"] = typed_value
                    metadata["editable"] = False

        # Re-inject search strategy options from code after DB merge,
        # since the DB stores options=null for this setting.
        if "search.search_strategy" in result:
            from local_deep_research.constants import get_available_strategies

            strategies = get_available_strategies()
            result["search.search_strategy"]["options"] = [
                {"label": s["label"], "value": s["name"]} for s in strategies
            ]

        return result

    def get_settings_snapshot(self, strict: bool = False) -> Dict[str, Any]:
        """
        Get a simplified settings snapshot with just key-value pairs.
        This is useful for passing settings to background threads or storing in metadata.

        Args:
            strict: If True, propagate database and enum query failures.

        Returns:
            Dictionary with setting keys mapped to their values
        """
        if self._closed:
            logger.error(
                "SettingsManager.get_settings_snapshot() called after close() — "
                "this is a bug; the caller should not reuse a closed manager",
            )
            # Fixed lifecycle text; no settings snapshot is included.
            # bearer:disable python_lang_exception
            raise RuntimeError(
                "SettingsManager has been closed. "
                "Create a new instance or call close() only at end of lifecycle."
            )

        all_settings = self.get_all_settings(strict=strict)
        settings_snapshot = {}

        for key, setting in all_settings.items():
            if isinstance(setting, dict) and "value" in setting:
                settings_snapshot[key] = setting["value"]
            else:
                settings_snapshot[key] = setting

        return settings_snapshot

    def create_or_update_setting(
        self, setting: Union[BaseSetting, Dict[str, Any]], commit: bool = True
    ) -> Optional[Setting]:
        """
        Create or update a setting

        Args:
            setting: Setting object or dictionary
            commit: Whether to commit the change

        Returns:
            The created or updated Setting model, or None if failed
        """
        if not self.db_session:
            logger.warning(
                "No database session available, cannot create/update setting"
            )
            return None
        if self.settings_locked:
            logger.error("Cannot edit settings because they are locked.")
            return None

        # Convert dict to BaseSetting if needed
        if isinstance(setting, dict):
            # Determine type from key if not specified
            if "type" not in setting and "key" in setting:
                setting_obj: BaseSetting
                key = setting["key"]
                if key.startswith("llm."):
                    setting_obj = LLMSetting(**setting)
                elif key.startswith("search."):
                    setting_obj = SearchSetting(**setting)
                elif key.startswith("report."):
                    setting_obj = ReportSetting(**setting)
                elif key.startswith("chat."):
                    setting_obj = ChatSetting(**setting)
                elif key.startswith("app."):
                    setting_obj = AppSetting(**setting)
                else:
                    # Keys outside the four buckets (e.g. local_search_*,
                    # embeddings.*, rag.*) live in their own namespaces.
                    # Use BaseSetting so the key is written verbatim —
                    # AppSetting's validator would otherwise prepend
                    # `app.` and silently relocate the row away from
                    # where every reader looks it up.  See #4208.
                    setting_obj = BaseSetting(type=SettingType.APP, **setting)
            else:
                # Use generic BaseSetting
                setting_obj = BaseSetting(**setting)
        else:
            setting_obj = setting

        if is_operator_only_setting(setting_obj.key):
            logger.bind(policy_audit=True).warning(
                "operator-only setting rejected at create_or_update_setting",
                key=setting_obj.key,
            )
            return None
        if self._is_environment_locked(
            setting_obj.key, "create_or_update_setting"
        ):
            return None

        try:
            # Check if setting exists
            db_setting = (
                self.db_session.query(Setting)
                .filter(Setting.key == setting_obj.key)
                .first()
            )

            if db_setting:
                # Update existing setting
                if not db_setting.editable:
                    logger.error(
                        "Cannot change setting '{}' because it "
                        "is marked as non-editable.",
                        setting_obj.key,
                    )
                    return None

                db_setting.value = setting_obj.value  # type: ignore[assignment]
                db_setting.name = setting_obj.name  # type: ignore[assignment]
                db_setting.description = setting_obj.description  # type: ignore[assignment]
                db_setting.category = setting_obj.category  # type: ignore[assignment]
                db_setting.type = setting_obj.type  # type: ignore[assignment]
                db_setting.ui_element = setting_obj.ui_element  # type: ignore[assignment]
                db_setting.options = setting_obj.options  # type: ignore[assignment]
                db_setting.min_value = setting_obj.min_value  # type: ignore[assignment]
                db_setting.max_value = setting_obj.max_value  # type: ignore[assignment]
                db_setting.step = setting_obj.step  # type: ignore[assignment]
                db_setting.visible = setting_obj.visible  # type: ignore[assignment]
                db_setting.editable = setting_obj.editable  # type: ignore[assignment]
                db_setting.updated_at = (  # type: ignore[assignment]
                    func.now()
                )  # Explicitly set the current timestamp
            else:
                # Refuse to CREATE a new row for a malformed key (see
                # is_valid_setting_key / #4840). Updates to existing rows
                # above are unaffected.
                if not is_valid_setting_key(setting_obj.key):
                    logger.error(
                        "Refusing to create setting with malformed key {!r}",
                        setting_obj.key,
                    )
                    return None

                # Create new setting
                db_setting = Setting(
                    key=setting_obj.key,
                    value=setting_obj.value,
                    type=setting_obj.type,
                    name=setting_obj.name,
                    description=setting_obj.description,
                    category=setting_obj.category,
                    ui_element=setting_obj.ui_element,
                    options=setting_obj.options,
                    min_value=setting_obj.min_value,
                    max_value=setting_obj.max_value,
                    step=setting_obj.step,
                    visible=setting_obj.visible,
                    editable=setting_obj.editable,
                )
                self.db_session.add(db_setting)

            if commit:
                self.db_session.commit()
                # Emit WebSocket event for settings change
                self._emit_settings_changed([setting_obj.key])

            return db_setting

        except SQLAlchemyError:
            logger.exception(
                f"Error creating/updating setting {setting_obj.key}"
            )
            self.db_session.rollback()
            return None

    def delete_setting(
        self, key: str, commit: bool = True, override_locked: bool = False
    ) -> bool:
        """
        Delete a setting

        Args:
            key: Setting key
            commit: Whether to commit the change
            override_locked: Delete even when settings are locked.

        Returns:
            True if successful, False otherwise
        """
        if not self.db_session:
            logger.warning(
                "No database session available, cannot delete setting"
            )
            return False

        if is_operator_only_setting(key):
            logger.bind(policy_audit=True).warning(
                "operator-only setting rejected at delete_setting", key=key
            )
            return False
        if self._is_environment_locked(key, "delete_setting"):
            return False

        if not override_locked and self.settings_locked:
            logger.bind(policy_audit=True).warning(
                "locked setting rejected at delete_setting", key=key
            )
            return False

        try:
            # Remove from database
            result = (
                self.db_session.query(Setting)
                .filter(Setting.key == key)
                .delete()
            )

            if commit:
                self.db_session.commit()

            return result > 0
        except SQLAlchemyError:
            logger.exception("Error deleting setting")
            self.db_session.rollback()
            return False

    def load_from_defaults_file(
        self,
        commit: bool = True,
        preserve_environment_locked: bool = False,
    ) -> None:
        """
        Import settings from the defaults settings file.

        Deliberately does NOT forward ``**kwargs`` to ``import_settings``:
        the passthrough exposed the settings-lock bypass
        (``override_locked=True``) through this wrapper, so a future
        request-side caller splatting user-controlled kwargs
        (``load_from_defaults_file(**payload)``) could have imported
        settings on a locked account (#5841). Only the two safe parameters
        are enumerated; any other keyword — ``override_locked`` included —
        raises ``TypeError``. The bootstrap/migration call sites that must
        bypass the lock call ``import_settings(..., override_locked=True)``
        directly (pinned by ``test_bootstrap_call_sites_pass_override_locked``).

        Args:
            commit: Whether to commit changes to database. The post-login
                atomic block in `web/routers/auth.py` performs its bulk
                default import through a direct ``import_settings(commit=False,
                override_locked=True)`` call and combines it with
                ``update_db_version(commit=False)`` under a single terminal
                ``db_session.commit()`` — preserving the all-or-nothing
                invariant is what prevents the sticky-loop bug where
                `app.version` is missing after a partial write.
            preserve_environment_locked: Passed to `import_settings` so a
                bulk reset/import cannot overwrite the stored values of
                settings with active ``LDR_*`` environment overrides.
        """
        start = time.perf_counter()
        row_count = len(self.default_settings)
        self.import_settings(
            self.default_settings,
            commit=commit,
            preserve_environment_locked=preserve_environment_locked,
        )
        elapsed_ms = (time.perf_counter() - start) * 1000
        if elapsed_ms > 100:
            logger.info(
                f"load_from_defaults_file imported {row_count} settings "
                f"in {elapsed_ms:.0f}ms (commit={commit})"
            )
        else:
            logger.debug(
                f"load_from_defaults_file imported {row_count} settings "
                f"in {elapsed_ms:.0f}ms (commit={commit})"
            )

    def db_version_matches_package(self) -> bool:
        """
        Returns:
            True if the version saved in the DB matches the package version.

        Reads the stored ``app.version`` row directly (``check_env=False``)
        rather than going through the env-aware read path. The schema-version
        marker must reflect what is persisted so a stale DB triggers a
        migration even when ``LDR_APP_VERSION`` happens to match the running
        package; reading the env value would mask the staleness and skip
        the migration.
        """
        db_version = self.get_setting("app.version", check_env=False)
        logger.debug(
            f"App version saved in DB is {db_version}, have package "
            f"settings from version {package_version}."
        )

        return bool(db_version == package_version)

    def update_db_version(self, commit: bool = True) -> None:
        """
        Updates the version saved in the DB based on the package version.

        Args:
            commit: Whether to commit the version write to the database.
                Callers that want to combine this with other writes into
                a single atomic transaction should pass commit=False and
                commit the session themselves. The post-login block in
                `web/auth/routes.py` relies on this to bundle the defaults
                import and the `app.version` write into one SQLite
                transaction; splitting them risks the sticky-loop state
                where `app.version` never gets written.
        """
        logger.debug(f"Updating saved DB version to {package_version}.")

        # Internal migration plumbing: this row is the schema-version
        # marker, not a user-facing setting. The env lock applies to
        # ordinary set/delete mutations via the public API; using the
        # env-guarded delete_setting() here would let a stale
        # LDR_APP_VERSION suppress the row replacement and either leave
        # the unique constraint violated (next commit raises) or leave
        # the stale marker in place. The narrowest correct path is to
        # delete the prior row directly and abort the call on failure
        # rather than blindly inserting a duplicate.
        if self.db_session is None:
            # Fixed lifecycle text; no session contents are included.
            # bearer:disable python_lang_exception
            raise RuntimeError("Database session is not initialized")
        try:
            self._check_thread_safety()
            self.db_session.query(Setting).filter(
                Setting.key == "app.version"
            ).delete(synchronize_session=False)
        except SQLAlchemyError:
            logger.exception("Error deleting prior app.version row")
            self.db_session.rollback()
            return

        version = Setting(
            key="app.version",
            value=package_version,
            description="Version of the app this database is associated with.",
            editable=False,
            name="App Version",
            type=SettingType.APP,
            ui_element="text",
            visible=False,
        )
        self.db_session.add(version)
        if commit:
            self.db_session.commit()

    def import_settings(
        self,
        settings_data: Dict[str, Any],
        commit: bool = True,
        overwrite: bool = True,
        delete_extra: bool = False,
        preserve_environment_locked: bool = False,
        override_locked: bool = False,
    ) -> None:
        """
        Import settings directly from the export format. This can be used to
        re-import settings that have been exported with `get_all_settings()`.

        Args:
            settings_data: The raw settings data to import.
            commit: Whether to commit the DB after loading the settings.
            overwrite: If true, it will overwrite the value of settings that
                are already in the database. If false, each key keeps the
                value of its own stored row; the values of its child keys
                are never folded into it. A stored value that is invalid
                for its type is kept as stored, with a warning that names
                the key but not the value, and the import continues.
            delete_extra: If true, it will delete any settings that are in
                the database but don't have a corresponding entry in
                `settings_data`.
            preserve_environment_locked: If true, preserve stored values for
                settings with active environment overrides while imported
                metadata is refreshed.
            override_locked: Import even when settings are locked. The
                bootstrap callers that add newly shipped defaults with
                `overwrite=False` pass this, since a locked account still
                has to receive settings introduced by an upgrade.

        """
        if not override_locked and self.settings_locked:
            logger.bind(policy_audit=True).warning(
                "locked settings rejected at import_settings"
            )
            return

        if self.db_session is None:
            # Fixed lifecycle text; no imported settings are included.
            # bearer:disable python_lang_exception
            raise RuntimeError("Database session is not initialized")
        logger.debug(f"Importing {len(settings_data)} settings")
        changed_keys: list[str] = []
        retained_keys: set[str] = set()
        # Schema-aware import (#5589) — see validate_imported_setting_value's
        # docstring for the trusted-vs-untrusted rationale (values retained
        # from the database are trusted here, not validated).
        defaults_for_import = self.default_settings
        try:
            # `overwrite=False` is the version-bump reconciliation point:
            # refresh JSON-defined schema while preserving the stored value.
            # Environment overrides must not become persisted values.
            for key, raw_setting_values in settings_data.items():
                if not is_valid_setting_key(key):
                    logger.warning(
                        "Skipping import of setting with malformed key {!r}",
                        key,
                    )
                    continue

                retained_keys.add(key)
                if is_operator_only_setting(key) and not (
                    override_locked and settings_data is self.default_settings
                ):
                    # User-driven import/reset must not replace or delete a
                    # legacy path. Upgrade/bootstrap imports use the exact
                    # trusted defaults mapping and reconcile its metadata
                    # with overwrite=False, retaining the stored location.
                    logger.bind(policy_audit=True).warning(
                        "operator-only setting rejected at import_settings",
                        key=key,
                    )
                    continue
                setting_values = dict(raw_setting_values)
                value_replaced_from_db = False
                if (
                    preserve_environment_locked
                    and check_env_setting(key) is not None
                ):
                    existing_value = (
                        self.db_session.query(Setting.value)
                        .filter(Setting.key == key)
                        .first()
                    )
                    if existing_value is not None:
                        logger.bind(policy_audit=True).warning(
                            "Preserving stored value for environment-locked "
                            "setting during import: {}",
                            key,
                        )
                        setting_values["value"] = existing_value[0]
                        value_replaced_from_db = True
                if not overwrite:
                    try:
                        existing_value = self.get_setting(key, check_env=False)
                        reread_own_row = isinstance(existing_value, dict)
                    except SettingsValueError:
                        # get_setting() also converts child rows, so the
                        # invalid value may belong to a child. Re-read the
                        # row stored under exactly this key.
                        reread_own_row = True
                    if reread_own_row:
                        # get_setting() folds child rows, including ASCII
                        # case variants its LIKE lookup matches, into a
                        # mapping. Writing that mapping back would replace
                        # this row's own value, so re-read the row stored
                        # under exactly this key.
                        stored_row = (
                            self.db_session.query(Setting)
                            .filter(Setting.key == key)
                            .first()
                        )
                        existing_value = None
                        if stored_row is not None:
                            try:
                                existing_value = self.__get_typed_setting_value(
                                    stored_row, None, check_env=False
                                )
                            except SettingsValueError as error:
                                # One invalid stored row must not abort the
                                # reconciliation for every other key, and
                                # replacing it with the default would hide
                                # the error. Keep the stored value; reads
                                # keep reporting it until it is corrected.
                                # The reason is a fixed string that never
                                # contains the value.
                                reason = error.reason
                                logger.warning(
                                    "Keeping the stored value of setting "
                                    "{!r} during defaults reconciliation: {}",
                                    key,
                                    reason,
                                )
                                existing_value = stored_row.value
                    if existing_value is not None:
                        setting_values["value"] = existing_value
                        value_replaced_from_db = True

                if not value_replaced_from_db:
                    # Schema-aware import (#5589): untrusted file value —
                    # see validate_imported_setting_value's docstring for
                    # the trusted-vs-untrusted rationale.
                    default_meta = defaults_for_import.get(key)
                    if default_meta is not None:
                        invalid_reason = validate_imported_setting_value(
                            key,
                            setting_values.get("value"),
                            default_meta,
                        )
                        if invalid_reason is not None:
                            # The value itself is never logged: an
                            # imported settings value is a plausible
                            # secret carrier. This mirrors the
                            # equivalent line in
                            # ``api/settings_utils.py``. ``invalid_reason``
                            # is built from the *defaults* metadata
                            # (min/max/options), never from the imported
                            # value.
                            logger.warning(
                                "Skipping import of setting {!r}: value of "
                                "type {} is invalid under the current "
                                "defaults schema — {}",
                                key,
                                type(setting_values.get("value")).__name__,
                                invalid_reason,
                            )
                            continue

                # Import needs strict failure semantics. The public
                # delete_setting() helper intentionally swallows SQL errors,
                # which would allow this transaction to continue after a
                # rollback and commit only a suffix of the import.
                (
                    self.db_session.query(Setting)
                    .filter(Setting.key == key)
                    .delete(synchronize_session=False)
                )

                if "type" in setting_values and isinstance(
                    setting_values["type"], str
                ):
                    setting_values["type"] = SettingType[setting_values["type"]]

                self.db_session.add(
                    Setting(key=key, **_filter_setting_columns(setting_values))
                )
                changed_keys.append(key)

            if delete_extra:
                existing_keys = [
                    str(row[0])
                    for row in self.db_session.query(Setting.key).all()
                ]
                for key in existing_keys:
                    if (
                        key in retained_keys
                        or is_operator_only_setting(key)
                        or (
                            preserve_environment_locked
                            and check_env_setting(key) is not None
                        )
                    ):
                        continue
                    logger.debug(f"Deleting extraneous setting: {key}")
                    (
                        self.db_session.query(Setting)
                        .filter(Setting.key == key)
                        .delete(synchronize_session=False)
                    )
                    changed_keys.append(key)

            if commit:
                self.db_session.commit()
                logger.info(
                    f"Successfully imported {len(changed_keys)} settings"
                )
                self._emit_settings_changed(changed_keys)
        except Exception:
            self.db_session.rollback()
            raise

    def emit_settings_changed_after_commit(
        self, changed_keys: List[str]
    ) -> None:
        """Notify the current user's clients after the caller commits settings."""
        self._emit_settings_changed(changed_keys)

    def _emit_settings_changed(self, changed_keys: Optional[List[Any]] = None):
        """
        Emit WebSocket event when settings change

        Args:
            changed_keys: List of setting keys that changed
        """
        try:
            # Import here to avoid circular imports
            from ..web.services.socketio_asgi import emit_to_user
            from ..security.data_sanitizer import DataSanitizer

            # Keep the event scoped to the owning user's browser tabs, never
            # every connected client on this shared Socket.IO server (#4437).
            # A change made outside a
            # request context (app start-up defaults, background workers,
            # migrations) has no user tab to notify, so we skip the emit
            # entirely rather than fall back to a broadcast.
            from datetime import datetime, UTC

            # Resolve the user from the contextvar set by DatabaseMiddleware.
            # SettingsManager itself does not hold a username attribute.
            from ..utilities.request_context import get_current_username

            username = get_current_username()
            if not username:
                logger.debug(
                    "Skipping settings_changed emit: no request username context"
                )
                return

            # Read the changed values only after the recipient check: each
            # read types the stored row, so an invalid value or unknown
            # ui_element raises for an emit that never happens.
            settings_data = {}
            if changed_keys:
                for key in changed_keys:
                    setting_value = self.get_setting(key)
                    if setting_value is not None:
                        # Match the HTTP read contract, including custom
                        # password controls whose key has no secret suffix.
                        # The exact DB row takes precedence over defaults.
                        exact_row = next(
                            (
                                row
                                for row in self.__query_settings(key)
                                if str(row.key) == key
                            ),
                            None,
                        )
                        if exact_row is not None:
                            ui_element = exact_row.ui_element
                        else:
                            default_meta = self.default_settings.get(key, {})
                            ui_element = (
                                default_meta.get("ui_element")
                                if isinstance(default_meta, dict)
                                else None
                            )
                        settings_data[key] = {
                            "value": DataSanitizer.redact_value(
                                key, ui_element, setting_value
                            )
                        }

            emit_to_user(
                "settings_changed",
                username,
                {
                    "changed_keys": changed_keys or [],
                    "settings": settings_data,
                    "timestamp": datetime.now(UTC).isoformat(),
                },
            )

            logger.debug(
                f"Emitted settings_changed event for keys: {changed_keys}"
            )

        except Exception:
            logger.exception("Failed to emit settings change event")
            # Don't let WebSocket emission failures break settings saving

    @staticmethod
    def get_bootstrap_env_vars() -> Dict[str, str]:
        """
        Get environment variables that must be available before database access.
        These are critical for system initialization.

        Returns:
            Dict mapping env var names to their descriptions
        """
        # Get bootstrap vars from env registry
        return env_registry.get_bootstrap_vars()

    @staticmethod
    def is_bootstrap_env_var(env_var: str) -> bool:
        """
        Check if an environment variable is a bootstrap variable (needed before DB access).

        Args:
            env_var: Environment variable name

        Returns:
            True if this is a bootstrap variable
        """
        bootstrap_vars = SettingsManager.get_bootstrap_env_vars()
        return env_var in bootstrap_vars

    @staticmethod
    def is_env_only_setting(key: str) -> bool:
        """
        Check if a setting key is environment-only.

        Args:
            key: Setting key to check

        Returns:
            True if it's an env-only setting, False otherwise
        """
        return env_registry.is_env_only(key)

    @staticmethod
    def get_env_var_for_setting(setting_key: str) -> str:
        """
        Get the environment variable name for a given setting key.

        Args:
            setting_key: Setting key (e.g., "app.host")

        Returns:
            Environment variable name (e.g., "LDR_APP_HOST")
        """
        # Use the same logic as check_env_setting for consistency
        return f"LDR_{'_'.join(setting_key.split('.')).upper()}"

    @staticmethod
    def get_setting_key_for_env_var(env_var: str) -> Optional[str]:
        """
        Get the setting key for a given environment variable.

        Args:
            env_var: Environment variable name (e.g., "LDR_APP_HOST")

        Returns:
            Setting key (e.g., "app.host") or None if not a valid LDR env var
        """
        if not env_var.startswith("LDR_"):
            return None

        # Remove LDR_ prefix and convert to lowercase
        without_prefix = env_var[4:]
        parts = without_prefix.split("_")

        return ".".join(part.lower() for part in parts)


class SnapshotSettingsContext:
    """Read-only settings context backed by a snapshot dict.

    Unwraps {"value": x} setting objects into plain values and provides
    get_setting(key, default) for thread-safe snapshot access.
    """

    def __init__(
        self, snapshot=None, username=None, missing_key_log_level="DEBUG"
    ):
        self.snapshot = snapshot or {}
        self.username = username
        self._missing_key_log_level = missing_key_log_level
        self.values = {}
        for key, setting in self.snapshot.items():
            if isinstance(setting, dict) and "value" in setting:
                self.values[key] = setting["value"]
            else:
                self.values[key] = setting

    def get_setting(self, key, default=None):
        """Return the setting value for *key*, or *default* if absent."""
        if key in self.values:
            return self.values[key]
        logger.log(
            self._missing_key_log_level,
            "Setting '{}' not found in snapshot, using default",
            key,
        )
        return default