~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
import functools
import json
import math
import os
import string
import threading
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
from loguru import logger
from sqlalchemy import and_, func, or_
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from .. import defaults
from ..__version__ import __version__ as package_version
from ..database.models import Setting, SettingType
from ..utilities.sql_utils import escape_like
from ..utilities.type_utils import to_bool, to_positive_int
from ..web.models.settings import (
AppSetting,
BaseSetting,
ChatSetting,
LLMSetting,
ReportSetting,
SearchSetting,
)
from .base import ISettingsManager
from .env_registry import registry as env_registry
from .exceptions import SettingsValueError
# Existing saved values remain in place, but accounts cannot redirect library
# reads or writes through their settings after this release.
OPERATOR_ONLY_SETTINGS = frozenset({"research_library.storage_path"})
# SQLite's LIKE ignores case for ASCII letters only. get_setting() finds child
# rows with LIKE, so "RESEARCH_LIBRARY.STORAGE_PATH.x" is a child of
# "research_library.storage_path" there.
_ASCII_CASE_FOLD = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
def is_operator_only_setting(key: str) -> bool:
"""Whether only trusted defaults or an LDR_* environment override may set key.
Matches each operator-only key and its dotted child keys, ignoring ASCII
case the same way SQLite's LIKE does in the child-row lookup.
"""
folded = key.translate(_ASCII_CASE_FOLD)
for root in OPERATOR_ONLY_SETTINGS:
folded_root = root.translate(_ASCII_CASE_FOLD)
if folded == folded_root or folded.startswith(f"{folded_root}."):
return True
return False
def parse_boolean(value: Any) -> bool:
"""
Convert various representations to boolean using HTML checkbox semantics.
This function handles form values, JSON booleans, and environment variables,
ensuring consistent behavior across client and server.
**HTML Checkbox Semantics** (INTENTIONAL DESIGN):
- **Any value present (except explicit false) = checked = True**
- This matches standard HTML form behavior where checkbox presence indicates checked state
- In HTML forms, checkboxes send a value when checked, nothing when unchecked
**Examples**:
parse_boolean("on") # True - standard HTML checkbox value
parse_boolean("true") # True - explicit true
parse_boolean("1") # True - numeric true
parse_boolean("enabled") # True - any non-empty string
parse_boolean("disabled") # True - INTENTIONAL: any string = checkbox was checked!
parse_boolean("custom") # True - custom checkbox value
parse_boolean("false") # False - explicit false
parse_boolean("off") # False - explicit false
parse_boolean("0") # False - explicit false
parse_boolean("") # False - empty string = unchecked
parse_boolean(None) # False - missing = unchecked
**Why "disabled" returns True**:
This is NOT a bug! If a checkbox sends the value "disabled", it means the checkbox
was checked (present in form data). The actual string content doesn't matter for
HTML checkboxes - only presence vs absence matters.
Args:
value: Value to convert to boolean. Accepts strings, booleans, or None.
Returns:
bool: True for truthy values (any non-empty string except explicit false);
False for falsy values ('off', 'false', '0', '', 'no', False, None)
Note:
This function implements HTML form semantics, NOT generic boolean parsing.
See tests/settings/test_boolean_parsing.py for comprehensive test coverage.
"""
# Constants for boolean value parsing
FALSY_VALUES = ("off", "false", "0", "", "no")
# Handle already-boolean values
if isinstance(value, bool):
return value
# Handle None (missing values)
if value is None:
return False
# Handle string values
if isinstance(value, str):
value_lower = value.lower().strip()
# Explicitly falsy values (empty string, false-like values)
if value_lower in FALSY_VALUES:
return False
# Any other non-empty string = True (HTML checkbox semantics)
return True
# For other types (numbers, lists, etc.), use Python's bool conversion
return bool(value)
def _parse_number(x):
"""Parse number, returning int if it's a whole number, otherwise float."""
if isinstance(x, bool):
# Fixed validation text; no configured value is included.
# bearer:disable python_lang_exception
raise ValueError("Expected a number")
f = float(x)
if not math.isfinite(f):
# Fixed validation text; no configured value is included.
# bearer:disable python_lang_exception
raise ValueError("Expected a finite number")
if f.is_integer():
return int(f)
return f
def _parse_json_value(x):
"""Parse JSON ui_element values.
DB values (via SQLAlchemy JSON column) arrive as Python objects already.
Form POST and env var overrides arrive as raw strings and need parsing.
For example, a textarea containing ``["general"]`` arrives as the string
``'[\\r\\n "general"\\r\\n]'`` which must be decoded into a list.
"""
if isinstance(x, str):
stripped = x.strip()
if stripped:
try:
return json.loads(stripped)
except (json.JSONDecodeError, ValueError, RecursionError):
# The JSON database column may already contain a decoded
# scalar string. Preserve that value; environment input is
# known to be encoded and is parsed strictly by the reader.
return x
return x
def _parse_multiselect(x):
"""Parse multiselect value, handling both lists and strings.
DB values (via SQLAlchemy JSON column) arrive as Python lists already.
Env var overrides arrive as strings and need parsing — either as JSON
arrays (e.g. '["markdown","latex"]') or comma-separated values
(e.g. 'markdown,latex').
"""
if isinstance(x, list):
return x
if isinstance(x, str):
stripped = x.strip()
if stripped.startswith(("[", "{")):
parsed = json.loads(stripped)
if not isinstance(parsed, list):
# Fixed validation text; the parsed object is not included.
# bearer:disable python_lang_exception
raise ValueError("Expected a list")
return parsed
# Comma-separated values are an explicitly supported input format.
return [item.strip() for item in stripped.split(",") if item.strip()]
if x is None:
return None
# Fixed validation text; the rejected carrier is not included.
# bearer:disable python_lang_exception
raise ValueError("Expected a list or comma-separated values")
def _filter_setting_columns(data: dict) -> dict:
"""Filter a dict to only keys that are valid Setting model columns.
Prevents crashes when default_settings.json contains keys not present
as columns on the Setting model (e.g. future flags).
"""
valid_columns = {c.name for c in Setting.__table__.columns}
return {k: v for k, v in data.items() if k in valid_columns}
_SETTING_METADATA_FIELDS = (
"value",
"type",
"name",
"description",
"category",
"ui_element",
"options",
"min_value",
"max_value",
"step",
"visible",
"editable",
)
# Select settings whose options lists are populated dynamically (at code or
# runtime) rather than from the static defaults JSON. Their static options
# are only a sample of what is legal, so options validation must be skipped
# for them. Re-exported by `web/services/settings_service.py` (single
# source of truth) and imported by tests that assert identity across
# import orders.
DYNAMIC_SETTINGS = ["llm.provider", "llm.model", "search.tool"]
# Re-exported by `api/settings_utils.py` (single source of truth) and
# imported by tests that assert identity across import orders.
def validate_imported_setting_value(
key: str, value: Any, default_meta: Dict[str, Any]
) -> Optional[str]:
"""Validate a file-supplied setting value against the current-defaults
schema for its key.
Mirrors the runtime checks `web.services.settings_service
.validate_setting` applies on the settings save path (type coercion via
``get_typed_setting_value``, options / min / max constraints), but reads
constraints from the current defaults metadata instead of a stored
``Setting`` row. Used by ``import_settings`` so a pre-upgrade export
cannot resurrect values that are invalid under the current schema
(#5589).
Trusted-vs-untrusted rationale (the call-site comments in both
managers' ``import_settings`` point here): file-supplied values are
new input, so ``import_settings`` runs this check on them (keys with
no defaults metadata, and in-memory entries without a ``value``
field, are stored unchecked). Retained database or in-memory values
— kept through ``overwrite=False`` or
``preserve_environment_locked=True`` to preserve existing user data
— are not new input and deliberately skip this check. Re-validating
them would not discard them: on a reject, both managers keep the
key's existing row or entry, as they do for a rejected file value.
It would cost the key's metadata refresh instead: where retained
keys get refreshed metadata (the database manager, and env-locked
keys in the in-memory one), a key whose stored value fails the
current rules would keep its previous metadata (``ui_element``,
options, min/max). "Trusted" in the call-site comments refers to this
preservation decision; it does not imply prior schema validation.
Programmatic setters can bypass this check, and older values may
not satisfy the current defaults schema.
Returns ``None`` when the value is acceptable, or a human-readable
reason string when it is not.
"""
if key == "search.fetch.summary_input_max_length":
try:
to_positive_int(value)
except ValueError:
return "Value must be a positive integer"
ui_element = default_meta.get("ui_element")
if (
not isinstance(ui_element, str)
or ui_element not in UI_ELEMENT_TO_SETTING_TYPE
):
return "Unsupported setting type"
if value == default_meta.get("value"):
# The value equals the currently shipped default, so the defaults
# file itself vouches for it. Without this guard, a shipped default
# that is technically outside its own constraint metadata (e.g. a
# null optional numeric default) would be rejected and the key
# dropped on fresh-install seeding.
#
# ``app.theme`` used to be the example here: it shipped as ``dark``
# while the theme registry, which replaces that setting's options at
# runtime, had no ``dark`` entry. This guard is what let that invalid
# default seed successfully — and therefore what kept it invisible
# until the no-JS Save path validated it and reported the user's
# untouched theme as "failing". The current default, ``hashed``,
# is the registry ID of the existing Hashed dark palette, selected
# while light themes still need refinement.
return None
try:
typed_value = get_typed_setting_value(
key=key,
value=value,
ui_element=ui_element,
default=None,
check_env=False,
)
except SettingsValueError as error:
return error.reason
if typed_value is None:
# A shipped optional default may legitimately be null; the equality
# guard above accepts that exact value. Otherwise this is either an
# unconvertible input or a non-default null. The save path rejects it
# for typed controls, so imports must do the same rather than storing
# a value that the UI could not save.
if ui_element == "checkbox":
return "Value must be a boolean"
if ui_element in ("number", "slider", "range"):
return "Value must be a number"
return None
if ui_element == "checkbox":
if not isinstance(typed_value, bool):
return "Value must be a boolean"
elif ui_element in ("number", "slider", "range"):
if not isinstance(typed_value, (int, float)):
return "Value must be a number"
min_value = default_meta.get("min_value")
max_value = default_meta.get("max_value")
if min_value is not None and typed_value < min_value:
return f"Value must be at least {min_value}"
if max_value is not None and typed_value > max_value:
return f"Value must be at most {max_value}"
elif ui_element == "select":
options = default_meta.get("options")
# Skip options validation for dynamically populated dropdowns —
# their static options are only a sample of legal values.
if options and key not in DYNAMIC_SETTINGS:
allowed_values = [
opt.get("value") if isinstance(opt, dict) else opt
for opt in list(options)
]
if typed_value not in allowed_values:
return "Value must be one of: " + ", ".join(
str(v) for v in allowed_values
)
return None
_POLICY_AUDIT_KEYS = frozenset(
{
"llm.require_local_endpoint",
"llm.allowed_local_hostnames",
"embeddings.require_local",
}
)
def _is_policy_setting(key: str) -> bool:
"""Return True for security-relevant setting keys that need an
audit-log entry on change. Scope is intentionally narrow so this
audit hook doesn't widen into general settings-change logging.
Narrow is not the same as enumerated: the ``policy.`` arm is a PREFIX
match, not an allowlist, so this covers every current and future
``policy.*`` key including ones a caller creates. See
``_policy_audit_value`` for why the audit line therefore cannot assume
the values it records are scalars from a fixed vocabulary.
"""
if key.startswith("policy."):
return True
return key in _POLICY_AUDIT_KEYS
def _policy_audit_allowed_values(
default_meta: Optional[Dict[str, Any]],
) -> Optional[List[Any]]:
"""The fixed vocabulary a policy setting's value must come from, or
``None`` when the key has no such vocabulary.
Only a ``select``/``radio`` row with a static ``options`` list has one.
Used by ``_policy_audit_value`` to decide whether a STRING value is
safe to record verbatim: a value drawn from a shipped option list is a
known token (``adaptive``, ``strict``, ...), not free-form input, so it
cannot be a credential.
"""
if not isinstance(default_meta, dict):
return None
if default_meta.get("ui_element") not in ("select", "radio"):
return None
options = default_meta.get("options")
if not options:
return None
return [
opt.get("value") if isinstance(opt, dict) else opt
for opt in list(options)
]
def _policy_audit_value(
value: Any, allowed_values: Optional[List[Any]] = None
) -> str:
"""Render a value for the ``policy.*`` audit line so the line itself
can never become a credential-disclosure channel.
Recording old/new is the whole point of an audit entry, so this does
not drop the information -- it drops the part that can carry a secret.
The audit hook's scope was previously described as "``policy.*`` plus
three keys holding booleans and a hostname list", which is not what the
code does: ``_is_policy_setting`` PREFIX-matches ``policy.``, and three
keys already in scope ship as free-form, editable JSON with no item
schema at all -- ``policy.trusted_search_engines``,
``policy.trusted_inference_providers`` and
``llm.allowed_local_hostnames`` are all ``ui_element="json"``,
``editable=True``, default ``[]``. Their egress validator
(``validate_trusted_search_engines``) ``continue``s past any non-``str``
entry rather than rejecting it, so a list like
``[{"api_key": "sk-...", "name": "x"}]`` is accepted and reaches
``set_setting`` -- which then wrote both ``old=`` and ``new=`` to the
log in plaintext at WARNING. That line also sits OUTSIDE the
``if commit:`` block, so the bulk save routes (``commit=False``) reach
it too; it is not a PUT-only path (#6201 round 7).
So plaintext is kept ONLY where the value is PROVEN unable to carry
free-form text, and the proof is a check rather than a comment:
* ``None``/``bool``/``int``/``float`` -- no string content at all;
* a ``str`` that is one of the key's own registered ``select``
options (see ``_policy_audit_allowed_values``) -- a fixed
vocabulary shipped in ``defaults/``, e.g. ``policy.egress_scope``.
Everything else -- a free-form string, a dict, a list, any other
type -- is recorded as STRUCTURAL metadata only: the type name, plus
the length when it is a string or the entry count when it is a
container. No token derived from the value's CONTENT appears in the
rendering. An earlier version logged a truncated SHA-256 of the
value's canonical JSON; a stable unkeyed digest -- even cut to 64
bits, re-truncated, or keyed with a key shipped in the repository --
is an offline dictionary oracle: anyone holding the log can confirm
a guessed credential by hashing candidates offline, which is exactly
the disclosure this rendering exists to prevent (#6201 round 8: a
common password was recovered from the logged prefix by an offline
candidate list).
The cost of structure-only recording is honest and deliberate:
change detection on free-form values is COARSE. "still an
11-character string" / "the trusted-engine list went from 2 entries
to 3" is all an auditor gets; two different 11-character secrets
render IDENTICALLY, so the line can never confirm which candidate is
the real value. A string's length narrows a candidate list but
confirms nothing (every same-length candidate matches), which is
what separates it from a digest. Scalars from a fixed vocabulary and
values with no string content stay verbatim, so the policy decisions
themselves -- the actual subject of this audit -- remain fully
observable.
"""
if value is None or isinstance(value, (bool, int, float)):
return repr(value)
if allowed_values and value in allowed_values:
return repr(value)
if isinstance(value, (dict, list)):
return f"<{type(value).__name__}[{len(value)}]>"
if isinstance(value, str):
return f"<str[{len(value)}]>"
return f"<{type(value).__name__}>"
def is_valid_setting_key(key: Any) -> bool:
"""Return True if *key* is a well-formed setting key.
A valid key is a non-empty string of dot-separated segments where each
segment is non-empty and contains no whitespace or ASCII control characters.
This rejects the
malformed keys that corrupt prefix/namespace lookups (see #4840) — a
trailing dot ``"foo."``, a leading dot ``".foo"``, an empty segment
``"foo..bar"``, a blank key — as well as keys carrying whitespace
(``" foo"``, ``"foo. bar"``, ``"llm.o k"``), which would also break the
``key.split(".")`` → ``LDR_...`` env-var mapping. A malformed row makes
``get_setting("foo")`` return a ``{"": value}`` wrapper dict, which the
UI renders as ``[object Object]``.
Used on both sides: writes reject new malformed keys, and both read paths
(``SettingsManager.get_setting`` on the DB and ``get_setting_from_snapshot``
on a settings snapshot) filter malformed rows out of a prefix read so a
stray legacy row can never wrap a leaf value into an ``[object Object]``
dict.
"""
if not isinstance(key, str) or not key:
return False
return all(
seg
and not any(c.isspace() or ord(c) < 0x20 or ord(c) == 0x7F for c in seg)
for seg in key.split(".")
)
def _infer_ui_element(value: Any, current: str = "text") -> str:
"""Infer the appropriate ui_element string from a Python value's type.
Args:
value: The value to infer the ui_element from.
current: The existing ui_element. If it is already something more
specific than ``"text"``, it is kept as-is.
"""
if current != "text":
return current
if isinstance(value, bool):
return "checkbox"
if isinstance(value, (int, float)):
return "number"
if isinstance(value, (list, dict)):
return "json"
return "text"
# Default categories for each typed setting prefix, used by the self-heal
# block in set_setting() when a row's type column doesn't match its key
# prefix. The values mirror the canonical strings in
# web/routers/settings.py: a legacy chat.* row with type=APP and a
# stale category gets repointed to type=CHAT + category="chat" on next
# save. We use the most-general category per prefix here — sub-classifying
# llm_general vs llm_parameters depends on the specific key, but the
# self-heal only fires when the row was already mis-typed, so over-
# generalizing the category is preferable to leaving it stale.
_INFERRED_CATEGORY: Dict[str, str] = {
"llm.": "llm_general",
"search.": "search_general",
"report.": "report_parameters",
"database.": "database_parameters",
"chat.": "chat",
}
UI_ELEMENT_TO_SETTING_TYPE: Dict[str, Callable[..., Any]] = {
"text": str,
"json": _parse_json_value,
"password": str,
"select": str,
"number": _parse_number,
"range": _parse_number, # Same behavior as number for consistency
"checkbox": parse_boolean,
"textarea": str,
"multiselect": _parse_multiselect,
}
def _reject_json_constant(_value: str):
"""Reject JSON's non-standard NaN/Infinity extensions without echoing input."""
# The rejected argument is unused; this diagnostic is constant.
# bearer:disable python_lang_exception
raise ValueError("Expected a finite JSON number")
def _parse_environment_boolean(value: str) -> bool:
"""Environment input has no HTML checkbox presence semantics."""
normalized = value.strip().lower()
if normalized in ("true", "1", "yes", "on", "enabled"):
return True
if normalized in ("false", "0", "no", "off", "disabled"):
return False
# Fixed validation text; the environment value is not included.
# bearer:disable python_lang_exception
raise ValueError("Expected a boolean token")
def get_typed_setting_value(
key: str,
value: Any,
ui_element: str,
default: Any = None,
check_env: bool = True,
) -> Any:
"""
Extracts the value for a particular setting, ensuring that it has the
correct type.
Args:
key: The setting key.
value: The setting value from the database.
ui_element: The setting UI element ID.
default: Default value to return when the setting is unset (None).
check_env: If true, it will check the environment variable for
this setting before reading from the DB.
Returns:
The typed value. The separate fetch-summary budget preserves an
invalid value, with a warning, until a summary fetch rejects it.
Raises:
SettingsValueError: The selected value cannot be converted, or its
declared setting type is unsupported. Invalid overrides never
fall through to a lower-priority value.
"""
return _convert_setting_value(
key,
value,
ui_element,
default,
check_env_setting(key) if check_env else None,
)
def _convert_setting_value(
key: str,
value: Any,
ui_element: str,
default: Any,
env_value: str | None,
) -> Any:
"""Convert a selected value using an already resolved environment source."""
source = "environment" if env_value is not None else "value"
if (
not isinstance(ui_element, str)
or ui_element not in UI_ELEMENT_TO_SETTING_TYPE
):
# Report only the setting key, source and this fixed reason.
# bearer:disable python_lang_exception
raise SettingsValueError(
key, source=source, reason="unsupported setting type"
)
setting_type = UI_ELEMENT_TO_SETTING_TYPE[ui_element]
if env_value is not None:
value = env_value
if key == "search.fetch.summary_input_max_length":
# This exact positive integer is validated again before a summary
# fetch. Keep a malformed budget visible without blocking unrelated
# research modes or the settings page used to repair it. All other
# settings and unsupported UI types retain the strict read contract.
try:
return to_positive_int(value)
except ValueError:
pass
location = (
f"LDR_{'_'.join(key.split('.')).upper()}"
if env_value is not None
else f"Setting {key}"
)
logger.warning(
"{} must be a positive integer (value omitted). Summary-mode "
"fetches will fail until it is corrected.",
location,
)
return value
if value is None:
return default
try:
if ui_element == "json" and env_value is not None:
return json.loads(value, parse_constant=_reject_json_constant)
if ui_element == "checkbox" and env_value is not None:
# Environment variables are configuration, not submitted HTML
# checkboxes. An unrecognized token must not enable a feature.
return _parse_environment_boolean(value)
return setting_type(value)
except (ValueError, TypeError, OverflowError, RecursionError):
pass
# Raise outside the handler: even a suppressed __context__ can contain
# credentials from float()/json.loads() errors. See #6201.
# The control type is allowlisted above; no value or parser error is used.
# bearer:disable python_lang_exception
raise SettingsValueError(
key, source=source, reason=f"expected a valid {ui_element} value"
)
def check_env_setting(key: str) -> str | None:
"""
Checks environment variables for a particular setting.
Args:
key: The database key for the setting.
Returns:
The setting from the environment variables, or None if the variable
is not set or is empty.
Note:
Empty environment variables ("") are treated as unset. This is standard
practice across the ecosystem — see CPython's official docs (PYTHON*
env vars require "a non-empty string"), botocore PR #1687, Pallets/Click
PR #2223, and Vercel Turborepo PR #6929. Orchestration tools like Unraid,
Terraform, and Kubernetes manifests often cannot conditionally omit env
var declarations, so they pass "" for unconfigured values. Treating ""
as unset prevents these empty strings from overriding database defaults.
See: https://github.com/LearningCircuit/local-deep-research/pull/3362
"""
env_variable_name = f"LDR_{'_'.join(key.split('.')).upper()}"
env_value = os.getenv(env_variable_name)
# Treat empty string as unset — orchestration tools (Unraid, Terraform, K8s)
# often cannot omit env var declarations and pass "" for unconfigured values.
if env_value is not None and env_value != "":
logger.debug(f"Overriding {key} setting from environment variable.")
return env_value
if env_value == "":
logger.warning(
"Environment variable {} is set but empty — "
"ignoring it and falling back to DB/default for setting '{}'. "
"This is expected on Unraid or Docker templates that create "
"all variables even when left blank. To suppress this warning, "
"remove the variable from your environment or set a value.",
env_variable_name,
key,
)
return None
def apply_environment_overrides_to_snapshot(
snapshot: Dict[str, Any],
) -> Dict[str, Any]:
"""Return a copied snapshot with current LDR_* overrides reapplied.
Queued research snapshots can outlive a process restart. Environment values
are operator policy and retain highest precedence at actual dispatch time.
"""
if not isinstance(snapshot, dict):
return snapshot
effective = dict(snapshot)
# Old or partial queued snapshots may omit keys added after enqueue. Use
# the registered defaults as metadata so every active LDR_* override is
# injected at dispatch, not only overrides for keys already serialized.
default_metadata = SettingsManager().default_settings
candidate_keys = set(snapshot) | set(default_metadata)
# Function-local import: the binding module imports this one.
from ..config.openai_endpoint_binding import (
OPENAI_ENDPOINT_API_KEY_SETTING,
OPENAI_ENDPOINT_KEY_OWNER_MARKER,
)
if OPENAI_ENDPOINT_KEY_OWNER_MARKER in snapshot:
# Preserve the value associated with the recorded owner. Envelope
# reads already apply the current environment key at dispatch; it
# must not replace the configured value retained in the envelope.
# Raw entries are explicit or cleared values and stay unchanged.
candidate_keys.discard(OPENAI_ENDPOINT_API_KEY_SETTING)
for key in candidate_keys:
if check_env_setting(str(key)) is None:
continue
stored = snapshot.get(key, default_metadata.get(key))
metadata = stored if isinstance(stored, dict) else None
current = metadata.get("value") if metadata is not None else stored
if metadata is not None:
ui_element = str(metadata.get("ui_element", "text"))
elif isinstance(current, bool):
ui_element = "checkbox"
elif isinstance(current, (list, dict)):
ui_element = "json"
elif isinstance(current, (int, float)):
ui_element = "number"
else:
ui_element = "text"
typed = get_typed_setting_value(
str(key), current, ui_element, default=current, check_env=True
)
if metadata is not None:
updated = dict(metadata)
updated["value"] = typed
updated["editable"] = False
effective[key] = updated
else:
effective[key] = typed
return effective
class SettingsManager(ISettingsManager):
"""
Manager for handling application settings with database storage and file fallback.
Provides methods to get and set settings, with the ability to override settings in memory.
"""
def __init__(
self,
db_session: Optional[Session] = None,
owns_session: bool = False,
*,
seed_defaults: bool = True,
):
"""
Initialize the settings manager
Args:
db_session: SQLAlchemy session for database operations
owns_session: If True, close() will close the session.
Defaults to False (safe for borrowed sessions). Set to True
only when this manager created/owns the session — currently
only get_settings_manager() in db_utils.py does this.
seed_defaults: If True (the default), an empty settings table is
seeded from the defaults files and COMMITTED on
``db_session`` here. Pass False for a read-only manager on a
session that belongs to someone else's transaction (e.g. a
deletion reading the user's snapshot mid-transaction): reads
still merge the in-memory defaults, so an empty table reads
as all-defaults without writing anything.
"""
self.db_session = db_session
self._owns_session = owns_session
self._closed = False
self.db_first = True # Always prioritize DB settings
# Store the thread ID this instance was created in
self._creation_thread_id = threading.get_ident()
# Initialize settings lock as None - will be checked lazily
self.__settings_locked: Optional[bool] = None
# Auto-initialize settings if database is empty
if self.db_session and seed_defaults:
self._ensure_settings_initialized()
def close(self):
"""Close the DB session if this manager owns it.
Borrowed sessions (owns_session=False) are left open for their
owner to close (e.g. a caller that passed in its own db_session).
Safe to call multiple times — subsequent calls are no-ops.
"""
if self._owns_session and self.db_session is not None:
try:
logger.debug("Closing owned DB session in SettingsManager")
self.db_session.close()
except Exception:
logger.warning(
"Failed to close SettingsManager DB session — "
"connection may leak",
)
self._closed = True
self.db_session = None
def _ensure_settings_initialized(self):
"""Ensure settings are initialized in the database."""
# Check if we have any settings at all
from ..database.models import Setting
if self.db_session is None:
# Fixed lifecycle text; no session contents are included.
# bearer:disable python_lang_exception
raise RuntimeError("Database session is not initialized")
settings_count = self.db_session.query(Setting).count()
if settings_count == 0:
logger.info("No settings found in database, loading defaults")
# Trusted first-run seeding bypasses the lock via import_settings
# directly — the load_from_defaults_file wrapper no longer
# accepts override_locked (#5841). Time it directly too: routing
# through the wrapper used to log this for free.
start = time.perf_counter()
row_count = len(self.default_settings)
self.import_settings(
self.default_settings, commit=True, override_locked=True
)
elapsed_ms = (time.perf_counter() - start) * 1000
logger.info(
f"Default settings loaded successfully: {row_count} "
f"settings in {elapsed_ms:.0f}ms"
)
def _check_thread_safety(self):
"""Check if this instance is being used in the same thread it was created in."""
current_thread_id = threading.get_ident()
if self.db_session and current_thread_id != self._creation_thread_id:
# Only numeric IDs from threading.get_ident() are interpolated.
# bearer:disable python_lang_exception
raise RuntimeError(
f"SettingsManager instance created in thread {self._creation_thread_id} "
f"is being used in thread {current_thread_id}. This is not thread-safe! "
f"Create a new SettingsManager instance within the current thread context."
)
@property
def settings_locked(self) -> bool:
"""Check if settings are locked (lazy evaluation)."""
if self.__settings_locked is None:
try:
self.__settings_locked = parse_boolean(
self.get_setting("app.lock_settings", False)
)
if self.__settings_locked:
logger.info(
"Settings are locked. Disabling all settings changes."
)
except SettingsValueError:
raise
except Exception:
logger.bind(policy_audit=True).warning(
"Failed to check settings lock status; denying changes"
)
self.__settings_locked = True
return bool(self.__settings_locked)
def _is_environment_locked(self, key: str, operation: str) -> bool:
if check_env_setting(key) is None:
return False
logger.bind(policy_audit=True).warning(
"environment-locked setting rejected at {}", operation, key=key
)
return True
@functools.cached_property
def default_settings(self) -> Dict[str, Any]:
"""
Returns:
The default settings, loaded from JSON files and merged.
Automatically discovers and loads all .json files in the defaults
directory and its subdirectories.
Theme options are dynamically injected from the theme registry.
"""
settings: Dict[str, Any] = {}
try:
# Get the defaults package path
defaults_path = Path(defaults.__file__).parent
# Find all JSON files recursively in the defaults directory
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Only JSON defaults beneath the installed defaults package are enumerated
json_files = sorted(defaults_path.rglob("*.json"))
logger.debug(f"Found {len(json_files)} JSON settings files")
# Load and merge all JSON files
for json_file in json_files:
try:
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Each file comes from the installed defaults package JSON glob above
with open(json_file, "r", encoding="utf-8-sig") as f:
file_settings = json.load(f)
# Get relative path for logging
relative_path = json_file.relative_to(defaults_path)
# Warn about key conflicts
conflicts = set(settings.keys()) & set(file_settings.keys())
if conflicts:
logger.warning(
f"Keys {conflicts} from {relative_path} "
f"override existing values"
)
settings.update(file_settings)
logger.debug(f"Loaded {relative_path}")
except json.JSONDecodeError:
logger.exception(f"Invalid JSON in {json_file}")
except Exception:
logger.warning(f"Could not load {json_file}")
except Exception:
logger.warning("Error loading settings files")
# Inject dynamic theme options from theme registry
if "app.theme" in settings:
try:
from local_deep_research.web.themes import theme_registry
settings["app.theme"]["options"] = (
theme_registry.get_settings_options()
)
except ImportError:
# Theme registry not available, use static options from JSON
pass
# Inject search strategy options from code (single source of truth)
if "search.search_strategy" in settings:
from local_deep_research.constants import get_available_strategies
strategies = get_available_strategies()
settings["search.search_strategy"]["options"] = [
{"label": s["label"], "value": s["name"]} for s in strategies
]
logger.debug(f"Loaded {len(settings)} total settings")
return settings
def __get_typed_setting_value(
self,
setting: Setting,
default: Any = None,
check_env: bool = True,
) -> Any:
"""
Extracts the value for a particular setting, ensuring that it has the
correct type.
Args:
setting: The setting to get the value for.
default: Default value to return if the value of the setting is
invalid.
check_env: If true, it will check the environment variable for
this setting before reading from the DB.
Returns:
The value of the setting.
"""
return get_typed_setting_value(
str(setting.key),
setting.value,
str(setting.ui_element),
default=default,
check_env=check_env,
)
def __query_settings(self, key: str | None = None) -> List[Setting]:
"""
Abstraction for querying settings that also transparently handles
reading the default settings file if the DB is not enabled.
Args:
key: The key to read. If None, it will read everything.
Returns:
The settings it queried.
"""
if self.db_session:
self._check_thread_safety()
query = self.db_session.query(Setting)
if key is not None:
# This will find exact matches and any subkeys.
#
# The ``startswith`` arm is intentionally restricted to
# subkeys that have AT LEAST one character past the dot
# (``key LIKE 'foo.%' AND key != 'foo.'``). The bare
# ``LIKE 'foo.%'`` form silently also matches ``'foo.'``
# (because ``%`` matches zero or more chars), which would
# treat a malformed trailing-dot duplicate row as a
# subkey of ``foo`` and cause ``get_setting`` to return
# a 2-key dict (``{"": value, "foo": value}``) instead
# of the primitive. See embedding-settings page bug.
# escape_like: the requested key is user-supplied (e.g. via
# /settings/api/bulk?keys[]=...), so escape its LIKE wildcards
# or ``keys[]=%`` would match every dotted key and dump the
# whole table. The ``!= f"{key}."`` guard compares against the
# raw stored key, so it stays unescaped.
query = query.filter(
or_(
Setting.key == key,
and_(
Setting.key.like(
f"{escape_like(key)}.%", escape="\\"
),
Setting.key != f"{key}.",
),
)
)
return query.all()
logger.debug(
"DB is disabled, reading setting '{}' from defaults file.", key
)
settings = []
for candidate_key, setting in self.default_settings.items():
if key is None or (
candidate_key == key or candidate_key.startswith(f"{key}.")
):
settings.append(
Setting(
key=candidate_key, # gitleaks:allow
**_filter_setting_columns(setting),
)
)
return settings
def get_setting(
self, key: str, default: Any = None, check_env: bool = True
) -> Any:
"""
Get a setting value
Args:
key: Setting key
default: Default value if setting is not found
check_env: If true, it will check the environment variable for
this setting before reading from the DB.
Returns:
Setting value or default if not found. An environment override
for a key with no row and no declared type is returned as the
raw string.
Raises:
SettingsValueError: The selected value cannot be converted to its
declared type, or that type is unsupported.
"""
if self._closed:
logger.error(
"SettingsManager.get_setting('{}') called after close() — "
"this is a bug; the caller should not reuse a closed manager",
key,
)
# Fixed lifecycle text; no setting value is included.
# bearer:disable python_lang_exception
raise RuntimeError(
"SettingsManager has been closed. "
"Create a new instance or call close() only at end of lifecycle."
)
# First check if this is an env-only setting
if env_registry.is_env_only(key):
return env_registry.get_strict(key, default, check_env=check_env)
# If using database first approach and session available, check database
try:
settings = self.__query_settings(key)
# Drop malformed subkey rows (e.g. legacy "foo." / "foo.." keys)
# that slip past the SQL prefix filter. #4852 excludes only the
# exact single-trailing-dot row; a "foo.." / "foo. " row would
# otherwise turn a leaf read into a `{".": v}`-style wrapper dict
# rendered as `[object Object]` (#4840). The exact-match row is
# always kept so a direct read of a malformed key still works.
#
# An operator-only key is never read as a namespace. Its child
# rows, whether saved before the policy or matched by LIKE as an
# ASCII case variant, are dropped, so the read returns the exact
# row's value, the LDR_* override or the default, never a
# mapping.
exact_row_only = is_operator_only_setting(key)
settings = [
s
for s in settings
if str(s.key) == key
or (not exact_row_only and is_valid_setting_key(str(s.key)))
]
if len(settings) == 1 and str(settings[0].key) == key:
# Only an exact row is a bottom-level key. A lone child row
# still represents a namespace and must return a mapping,
# matching get_setting_from_snapshot().
return self.__get_typed_setting_value(
settings[0], default, check_env
)
if settings:
# This is a higher-level key.
settings_map = {}
for setting in settings:
output_key = str(setting.key).removeprefix(f"{key}.")
settings_map[output_key] = self.__get_typed_setting_value(
setting, default, check_env
)
return settings_map
except SQLAlchemyError:
logger.exception(f"Error retrieving setting {key} from database")
if key == "app.lock_settings":
raise
# Check env var before returning default (setting not in DB)
if check_env:
env_value = check_env_setting(key)
if env_value is not None:
default_meta = self.default_settings.get(key)
if default_meta and isinstance(default_meta, dict):
ui_element = default_meta.get("ui_element", "text")
return get_typed_setting_value(
key,
None,
ui_element,
default=default,
check_env=True,
)
# No type is declared, so there is nothing to validate the
# override against: return it unconverted, as before strict
# reads. Untyped keys such as app.api_rate_limit and
# app.enable_api are read on every /api/v1 request, and their
# consumers parse the string themselves. Typed keys still
# raise above on a value that cannot be converted.
logger.warning(
"Setting '{}' has env var override but is not in "
"defaults — returning raw string without type "
"conversion. Add this setting to a defaults JSON "
"file with a ui_element type to enable proper "
"type conversion.",
key,
)
return env_value
# Return default if not found
return default
def get_bool_setting(
self, key: str, default: bool = False, check_env: bool = True
) -> bool:
"""
Get a setting value as a boolean, handling string conversion.
Args:
key: Setting key
default: Default boolean value if setting is not found
check_env: If true, it will check the environment variable for
this setting before reading from the DB.
Returns:
Boolean value of the setting
"""
value = self.get_setting(key, default, check_env)
return to_bool(value, default)
def set_setting(self, key: str, value: Any, commit: bool = True) -> bool:
"""
Set a setting value
Args:
key: Setting key
value: Setting value
commit: Whether to commit the change
Returns:
True if successful, False otherwise
"""
if self._closed:
logger.error(
"SettingsManager.set_setting('{}') called after close() — "
"this is a bug; the caller should not reuse a closed manager",
key,
)
# Fixed lifecycle text; no setting value is included.
# bearer:disable python_lang_exception
raise RuntimeError(
"SettingsManager has been closed. "
"Create a new instance or call close() only at end of lifecycle."
)
if not self.db_session:
logger.error(
"Cannot edit setting {} because no DB was provided.", key
)
return False
if is_operator_only_setting(key):
logger.bind(policy_audit=True).warning(
"operator-only setting rejected at set_setting", key=key
)
return False
if self.settings_locked:
logger.error("Cannot edit setting {} because they are locked.", key)
return False
if self._is_environment_locked(key, "set_setting"):
return False
# Always update database if available
try:
self._check_thread_safety()
setting = (
self.db_session.query(Setting)
.filter(Setting.key == key)
.first()
)
# Capture old value for the policy-change audit log below.
old_value = setting.value if setting is not None else None
if setting:
if not setting.editable:
logger.error(
"Cannot change setting '{}' because it "
"is marked as non-editable.",
key,
)
return False
setting.value = value # type: ignore[assignment]
setting.updated_at = ( # type: ignore[assignment]
func.now()
) # Explicitly set the current timestamp
# Self-heal stale ui_element from before inference was added
setting.ui_element = _infer_ui_element(
value, setting.ui_element
)
# Self-heal stale type from before the prefix dispatch was
# added (e.g. legacy chat.* rows created with type=APP).
# Also re-points category to the canonical per-prefix
# value, since a row with the wrong type column was
# almost certainly created before category dispatch was
# in place either.
inferred_type: Optional[SettingType] = None
inferred_category: Optional[str] = None
for prefix, category in _INFERRED_CATEGORY.items():
if key.startswith(prefix):
if prefix == "llm.":
inferred_type = SettingType.LLM
elif prefix == "search.":
inferred_type = SettingType.SEARCH
elif prefix == "report.":
inferred_type = SettingType.REPORT
elif prefix == "database.":
inferred_type = SettingType.DATABASE
elif prefix == "chat.":
inferred_type = SettingType.CHAT
inferred_category = category
break
# Only self-heal when the key matches a known prefix. Keys
# outside the dispatch map (e.g. focused_iteration.*,
# langgraph_agent.* which ship as type=SEARCH) must keep their
# shipped type — defaulting to APP here would wrongly demote
# them on every edit.
if inferred_type is not None and setting.type != inferred_type:
setting.type = inferred_type # type: ignore[assignment]
if inferred_category is not None:
setting.category = inferred_category # type: ignore[assignment]
else:
# Refuse to CREATE a new row for a malformed key (e.g. a
# trailing-dot key like "foo."). Existing rows are still
# updatable above; this only blocks minting new corruption.
if not is_valid_setting_key(key):
logger.error(
"Refusing to create setting with malformed key {!r}",
key,
)
return False
# Determine setting type from key
setting_type = SettingType.APP
if key.startswith("llm."):
setting_type = SettingType.LLM
elif key.startswith("search."):
setting_type = SettingType.SEARCH
elif key.startswith("report."):
setting_type = SettingType.REPORT
elif key.startswith("database."):
setting_type = SettingType.DATABASE
elif key.startswith("chat."):
setting_type = SettingType.CHAT
# Infer ui_element from the value type
ui_element = _infer_ui_element(value)
# Create a new setting
new_setting = Setting(
key=key,
value=value,
type=setting_type,
name=key.split(".")[-1].replace("_", " ").title(),
ui_element=ui_element,
description=f"Setting for {key}",
)
self.db_session.add(new_setting)
if commit:
self.db_session.commit()
# Emit WebSocket event for settings change
self._emit_settings_changed([key])
# N16: audit log on policy.* / llm.require_local_endpoint /
# embeddings.require_local changes. Targeted scope — only
# security-relevant settings are logged, to avoid widening
# this PR into a general audit-log refactor.
#
# The values are rendered through ``_policy_audit_value``, NOT
# interpolated raw: ``policy.`` is a prefix match and three of
# the keys in scope hold free-form, user-editable JSON, so a
# raw ``old={}``/``new={}`` here is a plaintext credential sink
# (#6201 round 7). This line is also outside the ``if commit:``
# block above on purpose — the bulk save routes pass
# ``commit=False`` and must still be audited.
if _is_policy_setting(key):
try:
_default_meta = self.default_settings.get(key)
except Exception:
# An audit line must never be able to fail the write it
# is auditing; without the metadata every string simply
# falls back to the structural form.
_default_meta = None
_audit_options = _policy_audit_allowed_values(_default_meta)
logger.bind(policy_audit=True).warning(
"policy setting changed | key={} old={} new={}",
key,
_policy_audit_value(old_value, _audit_options),
_policy_audit_value(value, _audit_options),
)
return True
except SQLAlchemyError:
logger.exception(f"Error setting value for key: {key}")
self.db_session.rollback()
return False
def clear_cache(self):
"""Clear the settings cache."""
self.__dict__.pop("default_settings", None)
logger.debug("Settings cache cleared")
def get_all_settings(
self,
bypass_cache: bool = False,
include_environment_overrides: bool = True,
strict: bool = False,
) -> Dict[str, Any]:
"""
Get all settings, merging defaults with database values.
This ensures that new settings added to defaults.json automatically
appear in the UI without requiring a database reset.
Args:
bypass_cache: If True, bypass the cache and read directly from database
include_environment_overrides: If True, validate effective values
and overlay current LDR_* values. False exports stored values
and metadata unchanged, including invalid values for repair.
strict: If True, propagate database and enum query failures.
Returns:
Dictionary of all settings
Raises:
SettingsValueError: With ``include_environment_overrides``, any
effective value cannot be converted to its declared type, or
that type is unsupported.
"""
if self._closed:
logger.error(
"SettingsManager.get_all_settings() called after close() — "
"this is a bug; the caller should not reuse a closed manager",
)
# Fixed lifecycle text; no settings snapshot is included.
# bearer:disable python_lang_exception
raise RuntimeError(
"SettingsManager has been closed. "
"Create a new instance or call close() only at end of lifecycle."
)
result = {}
# Start with defaults so new settings are always included
for key, default_setting in self.default_settings.items():
result[key] = dict(default_setting)
# Override with database settings
try:
db_settings = self.__query_settings() if self.db_session else None
except (SQLAlchemyError, LookupError):
# LookupError fires when a row's `type` column holds an enum
# value that's no longer in `SettingType` (e.g. legacy 'CHAT'-
# typed rows from removed features). The previous handler only
# caught SQLAlchemyError, so a single stale row would crash the
# whole snapshot — and every caller of get_all_settings (the
# /settings/api endpoint, benchmark start, research start, MCP
# entry points, …) downstream of it. Falling back to defaults-
# only is strictly safer than crashing.
logger.exception(
"Error querying settings from database in get_all_settings"
)
if strict:
raise
db_settings = []
if db_settings is None:
# Project defaults directly to the same metadata shape as a row.
# Bulk reads do not need SQLAlchemy instrumentation or temporary
# Setting objects just to copy their attributes back into dicts.
selected_settings = (
(
str(key),
{
field: metadata.get(field)
for field in _SETTING_METADATA_FIELDS
},
)
for key, metadata in self.default_settings.items()
)
else:
selected_settings = (
(
str(setting.key),
{
"value": setting.value,
"type": setting.type,
"name": setting.name,
"description": setting.description,
"category": setting.category,
"ui_element": setting.ui_element,
"options": setting.options,
"min_value": setting.min_value,
"max_value": setting.max_value,
"step": setting.step,
"visible": setting.visible,
"editable": setting.editable,
},
)
for setting in db_settings
)
for key, metadata in selected_settings:
if key.startswith("_"):
continue # internal snapshot keys are run metadata, never rows
# Handle type field - it might be a string or an enum
setting_type = metadata["type"]
if hasattr(setting_type, "name"):
setting_type = setting_type.name
# Log if this is a custom setting not in defaults
if key not in result:
logger.debug(
f"Database contains custom setting not in "
f"defaults: {key} (type={setting_type}, "
f"category={metadata['category']})"
)
# Override the default with the full database row — value AND
# schema (options, description, name, constraints).
#
# This is deliberate, not a bug: the DB row is a self-contained
# snapshot. Schema is NOT read live from JSON on every call; it is
# reconciled from the JSON defaults only at version bump, via the
# direct `import_settings(overwrite=False, override_locked=True)`
# bootstrap calls in `database/initialize.py` and post-login in
# `web/routers/auth.py`). Every release bumps the package version, so
# JSON metadata changes reach a user on their next login after
# upgrade. Do NOT change this to overlay defaults schema on every
# read — it bypasses that version gate and breaks the snapshot
# invariant (clean export/import round-trip, no mid-session drift).
# See closed PR #2474 for the rejected schema/value-separation
# approach and the reasoning.
metadata["type"] = setting_type
if (
include_environment_overrides and self.settings_locked
) or is_operator_only_setting(key):
metadata["editable"] = False
result[key] = metadata
# Validate only after choosing the effective row. A valid database row
# may replace an obsolete default, and a valid environment override may
# replace an invalid stored value. Neither may hide an invalid selected
# value by silently substituting another setting.
if include_environment_overrides:
for key, metadata in result.items():
env_value = check_env_setting(key)
typed_value = _convert_setting_value(
key,
metadata.get("value"),
metadata.get("ui_element", "text"),
None,
env_value,
)
if env_value is not None:
metadata["value"] = typed_value
metadata["editable"] = False
# Re-inject search strategy options from code after DB merge,
# since the DB stores options=null for this setting.
if "search.search_strategy" in result:
from local_deep_research.constants import get_available_strategies
strategies = get_available_strategies()
result["search.search_strategy"]["options"] = [
{"label": s["label"], "value": s["name"]} for s in strategies
]
return result
def get_settings_snapshot(self, strict: bool = False) -> Dict[str, Any]:
"""
Get a simplified settings snapshot with just key-value pairs.
This is useful for passing settings to background threads or storing in metadata.
Args:
strict: If True, propagate database and enum query failures.
Returns:
Dictionary with setting keys mapped to their values
"""
if self._closed:
logger.error(
"SettingsManager.get_settings_snapshot() called after close() — "
"this is a bug; the caller should not reuse a closed manager",
)
# Fixed lifecycle text; no settings snapshot is included.
# bearer:disable python_lang_exception
raise RuntimeError(
"SettingsManager has been closed. "
"Create a new instance or call close() only at end of lifecycle."
)
all_settings = self.get_all_settings(strict=strict)
settings_snapshot = {}
for key, setting in all_settings.items():
if isinstance(setting, dict) and "value" in setting:
settings_snapshot[key] = setting["value"]
else:
settings_snapshot[key] = setting
return settings_snapshot
def create_or_update_setting(
self, setting: Union[BaseSetting, Dict[str, Any]], commit: bool = True
) -> Optional[Setting]:
"""
Create or update a setting
Args:
setting: Setting object or dictionary
commit: Whether to commit the change
Returns:
The created or updated Setting model, or None if failed
"""
if not self.db_session:
logger.warning(
"No database session available, cannot create/update setting"
)
return None
if self.settings_locked:
logger.error("Cannot edit settings because they are locked.")
return None
# Convert dict to BaseSetting if needed
if isinstance(setting, dict):
# Determine type from key if not specified
if "type" not in setting and "key" in setting:
setting_obj: BaseSetting
key = setting["key"]
if key.startswith("llm."):
setting_obj = LLMSetting(**setting)
elif key.startswith("search."):
setting_obj = SearchSetting(**setting)
elif key.startswith("report."):
setting_obj = ReportSetting(**setting)
elif key.startswith("chat."):
setting_obj = ChatSetting(**setting)
elif key.startswith("app."):
setting_obj = AppSetting(**setting)
else:
# Keys outside the four buckets (e.g. local_search_*,
# embeddings.*, rag.*) live in their own namespaces.
# Use BaseSetting so the key is written verbatim —
# AppSetting's validator would otherwise prepend
# `app.` and silently relocate the row away from
# where every reader looks it up. See #4208.
setting_obj = BaseSetting(type=SettingType.APP, **setting)
else:
# Use generic BaseSetting
setting_obj = BaseSetting(**setting)
else:
setting_obj = setting
if is_operator_only_setting(setting_obj.key):
logger.bind(policy_audit=True).warning(
"operator-only setting rejected at create_or_update_setting",
key=setting_obj.key,
)
return None
if self._is_environment_locked(
setting_obj.key, "create_or_update_setting"
):
return None
try:
# Check if setting exists
db_setting = (
self.db_session.query(Setting)
.filter(Setting.key == setting_obj.key)
.first()
)
if db_setting:
# Update existing setting
if not db_setting.editable:
logger.error(
"Cannot change setting '{}' because it "
"is marked as non-editable.",
setting_obj.key,
)
return None
db_setting.value = setting_obj.value # type: ignore[assignment]
db_setting.name = setting_obj.name # type: ignore[assignment]
db_setting.description = setting_obj.description # type: ignore[assignment]
db_setting.category = setting_obj.category # type: ignore[assignment]
db_setting.type = setting_obj.type # type: ignore[assignment]
db_setting.ui_element = setting_obj.ui_element # type: ignore[assignment]
db_setting.options = setting_obj.options # type: ignore[assignment]
db_setting.min_value = setting_obj.min_value # type: ignore[assignment]
db_setting.max_value = setting_obj.max_value # type: ignore[assignment]
db_setting.step = setting_obj.step # type: ignore[assignment]
db_setting.visible = setting_obj.visible # type: ignore[assignment]
db_setting.editable = setting_obj.editable # type: ignore[assignment]
db_setting.updated_at = ( # type: ignore[assignment]
func.now()
) # Explicitly set the current timestamp
else:
# Refuse to CREATE a new row for a malformed key (see
# is_valid_setting_key / #4840). Updates to existing rows
# above are unaffected.
if not is_valid_setting_key(setting_obj.key):
logger.error(
"Refusing to create setting with malformed key {!r}",
setting_obj.key,
)
return None
# Create new setting
db_setting = Setting(
key=setting_obj.key,
value=setting_obj.value,
type=setting_obj.type,
name=setting_obj.name,
description=setting_obj.description,
category=setting_obj.category,
ui_element=setting_obj.ui_element,
options=setting_obj.options,
min_value=setting_obj.min_value,
max_value=setting_obj.max_value,
step=setting_obj.step,
visible=setting_obj.visible,
editable=setting_obj.editable,
)
self.db_session.add(db_setting)
if commit:
self.db_session.commit()
# Emit WebSocket event for settings change
self._emit_settings_changed([setting_obj.key])
return db_setting
except SQLAlchemyError:
logger.exception(
f"Error creating/updating setting {setting_obj.key}"
)
self.db_session.rollback()
return None
def delete_setting(
self, key: str, commit: bool = True, override_locked: bool = False
) -> bool:
"""
Delete a setting
Args:
key: Setting key
commit: Whether to commit the change
override_locked: Delete even when settings are locked.
Returns:
True if successful, False otherwise
"""
if not self.db_session:
logger.warning(
"No database session available, cannot delete setting"
)
return False
if is_operator_only_setting(key):
logger.bind(policy_audit=True).warning(
"operator-only setting rejected at delete_setting", key=key
)
return False
if self._is_environment_locked(key, "delete_setting"):
return False
if not override_locked and self.settings_locked:
logger.bind(policy_audit=True).warning(
"locked setting rejected at delete_setting", key=key
)
return False
try:
# Remove from database
result = (
self.db_session.query(Setting)
.filter(Setting.key == key)
.delete()
)
if commit:
self.db_session.commit()
return result > 0
except SQLAlchemyError:
logger.exception("Error deleting setting")
self.db_session.rollback()
return False
def load_from_defaults_file(
self,
commit: bool = True,
preserve_environment_locked: bool = False,
) -> None:
"""
Import settings from the defaults settings file.
Deliberately does NOT forward ``**kwargs`` to ``import_settings``:
the passthrough exposed the settings-lock bypass
(``override_locked=True``) through this wrapper, so a future
request-side caller splatting user-controlled kwargs
(``load_from_defaults_file(**payload)``) could have imported
settings on a locked account (#5841). Only the two safe parameters
are enumerated; any other keyword — ``override_locked`` included —
raises ``TypeError``. The bootstrap/migration call sites that must
bypass the lock call ``import_settings(..., override_locked=True)``
directly (pinned by ``test_bootstrap_call_sites_pass_override_locked``).
Args:
commit: Whether to commit changes to database. The post-login
atomic block in `web/routers/auth.py` performs its bulk
default import through a direct ``import_settings(commit=False,
override_locked=True)`` call and combines it with
``update_db_version(commit=False)`` under a single terminal
``db_session.commit()`` — preserving the all-or-nothing
invariant is what prevents the sticky-loop bug where
`app.version` is missing after a partial write.
preserve_environment_locked: Passed to `import_settings` so a
bulk reset/import cannot overwrite the stored values of
settings with active ``LDR_*`` environment overrides.
"""
start = time.perf_counter()
row_count = len(self.default_settings)
self.import_settings(
self.default_settings,
commit=commit,
preserve_environment_locked=preserve_environment_locked,
)
elapsed_ms = (time.perf_counter() - start) * 1000
if elapsed_ms > 100:
logger.info(
f"load_from_defaults_file imported {row_count} settings "
f"in {elapsed_ms:.0f}ms (commit={commit})"
)
else:
logger.debug(
f"load_from_defaults_file imported {row_count} settings "
f"in {elapsed_ms:.0f}ms (commit={commit})"
)
def db_version_matches_package(self) -> bool:
"""
Returns:
True if the version saved in the DB matches the package version.
Reads the stored ``app.version`` row directly (``check_env=False``)
rather than going through the env-aware read path. The schema-version
marker must reflect what is persisted so a stale DB triggers a
migration even when ``LDR_APP_VERSION`` happens to match the running
package; reading the env value would mask the staleness and skip
the migration.
"""
db_version = self.get_setting("app.version", check_env=False)
logger.debug(
f"App version saved in DB is {db_version}, have package "
f"settings from version {package_version}."
)
return bool(db_version == package_version)
def update_db_version(self, commit: bool = True) -> None:
"""
Updates the version saved in the DB based on the package version.
Args:
commit: Whether to commit the version write to the database.
Callers that want to combine this with other writes into
a single atomic transaction should pass commit=False and
commit the session themselves. The post-login block in
`web/auth/routes.py` relies on this to bundle the defaults
import and the `app.version` write into one SQLite
transaction; splitting them risks the sticky-loop state
where `app.version` never gets written.
"""
logger.debug(f"Updating saved DB version to {package_version}.")
# Internal migration plumbing: this row is the schema-version
# marker, not a user-facing setting. The env lock applies to
# ordinary set/delete mutations via the public API; using the
# env-guarded delete_setting() here would let a stale
# LDR_APP_VERSION suppress the row replacement and either leave
# the unique constraint violated (next commit raises) or leave
# the stale marker in place. The narrowest correct path is to
# delete the prior row directly and abort the call on failure
# rather than blindly inserting a duplicate.
if self.db_session is None:
# Fixed lifecycle text; no session contents are included.
# bearer:disable python_lang_exception
raise RuntimeError("Database session is not initialized")
try:
self._check_thread_safety()
self.db_session.query(Setting).filter(
Setting.key == "app.version"
).delete(synchronize_session=False)
except SQLAlchemyError:
logger.exception("Error deleting prior app.version row")
self.db_session.rollback()
return
version = Setting(
key="app.version",
value=package_version,
description="Version of the app this database is associated with.",
editable=False,
name="App Version",
type=SettingType.APP,
ui_element="text",
visible=False,
)
self.db_session.add(version)
if commit:
self.db_session.commit()
def import_settings(
self,
settings_data: Dict[str, Any],
commit: bool = True,
overwrite: bool = True,
delete_extra: bool = False,
preserve_environment_locked: bool = False,
override_locked: bool = False,
) -> None:
"""
Import settings directly from the export format. This can be used to
re-import settings that have been exported with `get_all_settings()`.
Args:
settings_data: The raw settings data to import.
commit: Whether to commit the DB after loading the settings.
overwrite: If true, it will overwrite the value of settings that
are already in the database. If false, each key keeps the
value of its own stored row; the values of its child keys
are never folded into it. A stored value that is invalid
for its type is kept as stored, with a warning that names
the key but not the value, and the import continues.
delete_extra: If true, it will delete any settings that are in
the database but don't have a corresponding entry in
`settings_data`.
preserve_environment_locked: If true, preserve stored values for
settings with active environment overrides while imported
metadata is refreshed.
override_locked: Import even when settings are locked. The
bootstrap callers that add newly shipped defaults with
`overwrite=False` pass this, since a locked account still
has to receive settings introduced by an upgrade.
"""
if not override_locked and self.settings_locked:
logger.bind(policy_audit=True).warning(
"locked settings rejected at import_settings"
)
return
if self.db_session is None:
# Fixed lifecycle text; no imported settings are included.
# bearer:disable python_lang_exception
raise RuntimeError("Database session is not initialized")
logger.debug(f"Importing {len(settings_data)} settings")
changed_keys: list[str] = []
retained_keys: set[str] = set()
# Schema-aware import (#5589) — see validate_imported_setting_value's
# docstring for the trusted-vs-untrusted rationale (values retained
# from the database are trusted here, not validated).
defaults_for_import = self.default_settings
try:
# `overwrite=False` is the version-bump reconciliation point:
# refresh JSON-defined schema while preserving the stored value.
# Environment overrides must not become persisted values.
for key, raw_setting_values in settings_data.items():
if not is_valid_setting_key(key):
logger.warning(
"Skipping import of setting with malformed key {!r}",
key,
)
continue
retained_keys.add(key)
if is_operator_only_setting(key) and not (
override_locked and settings_data is self.default_settings
):
# User-driven import/reset must not replace or delete a
# legacy path. Upgrade/bootstrap imports use the exact
# trusted defaults mapping and reconcile its metadata
# with overwrite=False, retaining the stored location.
logger.bind(policy_audit=True).warning(
"operator-only setting rejected at import_settings",
key=key,
)
continue
setting_values = dict(raw_setting_values)
value_replaced_from_db = False
if (
preserve_environment_locked
and check_env_setting(key) is not None
):
existing_value = (
self.db_session.query(Setting.value)
.filter(Setting.key == key)
.first()
)
if existing_value is not None:
logger.bind(policy_audit=True).warning(
"Preserving stored value for environment-locked "
"setting during import: {}",
key,
)
setting_values["value"] = existing_value[0]
value_replaced_from_db = True
if not overwrite:
try:
existing_value = self.get_setting(key, check_env=False)
reread_own_row = isinstance(existing_value, dict)
except SettingsValueError:
# get_setting() also converts child rows, so the
# invalid value may belong to a child. Re-read the
# row stored under exactly this key.
reread_own_row = True
if reread_own_row:
# get_setting() folds child rows, including ASCII
# case variants its LIKE lookup matches, into a
# mapping. Writing that mapping back would replace
# this row's own value, so re-read the row stored
# under exactly this key.
stored_row = (
self.db_session.query(Setting)
.filter(Setting.key == key)
.first()
)
existing_value = None
if stored_row is not None:
try:
existing_value = self.__get_typed_setting_value(
stored_row, None, check_env=False
)
except SettingsValueError as error:
# One invalid stored row must not abort the
# reconciliation for every other key, and
# replacing it with the default would hide
# the error. Keep the stored value; reads
# keep reporting it until it is corrected.
# The reason is a fixed string that never
# contains the value.
reason = error.reason
logger.warning(
"Keeping the stored value of setting "
"{!r} during defaults reconciliation: {}",
key,
reason,
)
existing_value = stored_row.value
if existing_value is not None:
setting_values["value"] = existing_value
value_replaced_from_db = True
if not value_replaced_from_db:
# Schema-aware import (#5589): untrusted file value —
# see validate_imported_setting_value's docstring for
# the trusted-vs-untrusted rationale.
default_meta = defaults_for_import.get(key)
if default_meta is not None:
invalid_reason = validate_imported_setting_value(
key,
setting_values.get("value"),
default_meta,
)
if invalid_reason is not None:
# The value itself is never logged: an
# imported settings value is a plausible
# secret carrier. This mirrors the
# equivalent line in
# ``api/settings_utils.py``. ``invalid_reason``
# is built from the *defaults* metadata
# (min/max/options), never from the imported
# value.
logger.warning(
"Skipping import of setting {!r}: value of "
"type {} is invalid under the current "
"defaults schema — {}",
key,
type(setting_values.get("value")).__name__,
invalid_reason,
)
continue
# Import needs strict failure semantics. The public
# delete_setting() helper intentionally swallows SQL errors,
# which would allow this transaction to continue after a
# rollback and commit only a suffix of the import.
(
self.db_session.query(Setting)
.filter(Setting.key == key)
.delete(synchronize_session=False)
)
if "type" in setting_values and isinstance(
setting_values["type"], str
):
setting_values["type"] = SettingType[setting_values["type"]]
self.db_session.add(
Setting(key=key, **_filter_setting_columns(setting_values))
)
changed_keys.append(key)
if delete_extra:
existing_keys = [
str(row[0])
for row in self.db_session.query(Setting.key).all()
]
for key in existing_keys:
if (
key in retained_keys
or is_operator_only_setting(key)
or (
preserve_environment_locked
and check_env_setting(key) is not None
)
):
continue
logger.debug(f"Deleting extraneous setting: {key}")
(
self.db_session.query(Setting)
.filter(Setting.key == key)
.delete(synchronize_session=False)
)
changed_keys.append(key)
if commit:
self.db_session.commit()
logger.info(
f"Successfully imported {len(changed_keys)} settings"
)
self._emit_settings_changed(changed_keys)
except Exception:
self.db_session.rollback()
raise
def emit_settings_changed_after_commit(
self, changed_keys: List[str]
) -> None:
"""Notify the current user's clients after the caller commits settings."""
self._emit_settings_changed(changed_keys)
def _emit_settings_changed(self, changed_keys: Optional[List[Any]] = None):
"""
Emit WebSocket event when settings change
Args:
changed_keys: List of setting keys that changed
"""
try:
# Import here to avoid circular imports
from ..web.services.socketio_asgi import emit_to_user
from ..security.data_sanitizer import DataSanitizer
# Keep the event scoped to the owning user's browser tabs, never
# every connected client on this shared Socket.IO server (#4437).
# A change made outside a
# request context (app start-up defaults, background workers,
# migrations) has no user tab to notify, so we skip the emit
# entirely rather than fall back to a broadcast.
from datetime import datetime, UTC
# Resolve the user from the contextvar set by DatabaseMiddleware.
# SettingsManager itself does not hold a username attribute.
from ..utilities.request_context import get_current_username
username = get_current_username()
if not username:
logger.debug(
"Skipping settings_changed emit: no request username context"
)
return
# Read the changed values only after the recipient check: each
# read types the stored row, so an invalid value or unknown
# ui_element raises for an emit that never happens.
settings_data = {}
if changed_keys:
for key in changed_keys:
setting_value = self.get_setting(key)
if setting_value is not None:
# Match the HTTP read contract, including custom
# password controls whose key has no secret suffix.
# The exact DB row takes precedence over defaults.
exact_row = next(
(
row
for row in self.__query_settings(key)
if str(row.key) == key
),
None,
)
if exact_row is not None:
ui_element = exact_row.ui_element
else:
default_meta = self.default_settings.get(key, {})
ui_element = (
default_meta.get("ui_element")
if isinstance(default_meta, dict)
else None
)
settings_data[key] = {
"value": DataSanitizer.redact_value(
key, ui_element, setting_value
)
}
emit_to_user(
"settings_changed",
username,
{
"changed_keys": changed_keys or [],
"settings": settings_data,
"timestamp": datetime.now(UTC).isoformat(),
},
)
logger.debug(
f"Emitted settings_changed event for keys: {changed_keys}"
)
except Exception:
logger.exception("Failed to emit settings change event")
# Don't let WebSocket emission failures break settings saving
@staticmethod
def get_bootstrap_env_vars() -> Dict[str, str]:
"""
Get environment variables that must be available before database access.
These are critical for system initialization.
Returns:
Dict mapping env var names to their descriptions
"""
# Get bootstrap vars from env registry
return env_registry.get_bootstrap_vars()
@staticmethod
def is_bootstrap_env_var(env_var: str) -> bool:
"""
Check if an environment variable is a bootstrap variable (needed before DB access).
Args:
env_var: Environment variable name
Returns:
True if this is a bootstrap variable
"""
bootstrap_vars = SettingsManager.get_bootstrap_env_vars()
return env_var in bootstrap_vars
@staticmethod
def is_env_only_setting(key: str) -> bool:
"""
Check if a setting key is environment-only.
Args:
key: Setting key to check
Returns:
True if it's an env-only setting, False otherwise
"""
return env_registry.is_env_only(key)
@staticmethod
def get_env_var_for_setting(setting_key: str) -> str:
"""
Get the environment variable name for a given setting key.
Args:
setting_key: Setting key (e.g., "app.host")
Returns:
Environment variable name (e.g., "LDR_APP_HOST")
"""
# Use the same logic as check_env_setting for consistency
return f"LDR_{'_'.join(setting_key.split('.')).upper()}"
@staticmethod
def get_setting_key_for_env_var(env_var: str) -> Optional[str]:
"""
Get the setting key for a given environment variable.
Args:
env_var: Environment variable name (e.g., "LDR_APP_HOST")
Returns:
Setting key (e.g., "app.host") or None if not a valid LDR env var
"""
if not env_var.startswith("LDR_"):
return None
# Remove LDR_ prefix and convert to lowercase
without_prefix = env_var[4:]
parts = without_prefix.split("_")
return ".".join(part.lower() for part in parts)
class SnapshotSettingsContext:
"""Read-only settings context backed by a snapshot dict.
Unwraps {"value": x} setting objects into plain values and provides
get_setting(key, default) for thread-safe snapshot access.
"""
def __init__(
self, snapshot=None, username=None, missing_key_log_level="DEBUG"
):
self.snapshot = snapshot or {}
self.username = username
self._missing_key_log_level = missing_key_log_level
self.values = {}
for key, setting in self.snapshot.items():
if isinstance(setting, dict) and "value" in setting:
self.values[key] = setting["value"]
else:
self.values[key] = setting
def get_setting(self, key, default=None):
"""Return the setting value for *key*, or *default* if absent."""
if key in self.values:
return self.values[key]
logger.log(
self._missing_key_log_level,
"Setting '{}' not found in snapshot, using default",
key,
)
return default