~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python3
"""
Generate CONFIGURATION.md from default settings JSON files and env_definitions.
Usage:
python scripts/generate_config_docs.py # Write to docs/CONFIGURATION.md
python scripts/generate_config_docs.py --output /tmp/out # Write to custom location
python scripts/generate_config_docs.py --check # Exit 1 if docs are stale
"""
import argparse
import ast
import json
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional
def get_project_root() -> Path:
"""Return the project root directory."""
return Path(__file__).resolve().parent.parent
def get_env_var_name(key: str) -> str:
"""Convert setting key to environment variable name."""
return f"LDR_{key.replace('.', '_').upper()}"
def format_value(value: Any) -> str:
"""Format default value for markdown."""
if value is None:
return "null"
if isinstance(value, bool):
return str(value).lower()
if isinstance(value, (dict, list)):
return f"`{json.dumps(value)}`"
return str(value)
def _discover_env_definition_files(env_defs_dir: Path) -> List[Path]:
"""Auto-discover env_definitions modules, excluding __init__.py and env_settings.py."""
if not env_defs_dir.is_dir():
return []
return sorted(
p
for p in env_defs_dir.glob("*.py")
if p.name not in ("__init__.py", "env_settings.py")
)
def _category_from_filename(filename: str) -> str:
"""Derive a human-readable category name from a filename.
Example: 'db_config.py' -> 'Db Config'
"""
stem = filename.removesuffix(".py")
return stem.replace("_", " ").title()
def _extract_setting_from_call(node: ast.Call) -> Optional[Dict[str, Any]]:
"""Extract a setting dict from a *Setting() AST call node."""
if not (
isinstance(node.func, ast.Name) and node.func.id.endswith("Setting")
):
return None
keywords = {k.arg: k.value for k in node.keywords if k.arg}
if "key" not in keywords:
return None
key_node = keywords["key"]
if not isinstance(key_node, ast.Constant):
return None
key = key_node.value # gitleaks:allow
# Description — may be a simple string or a parenthesised concatenation
description = ""
if "description" in keywords:
desc_node = keywords["description"]
if isinstance(desc_node, ast.Constant):
description = desc_node.value
else:
description = ast.unparse(desc_node)
# Default
default_val = "None"
if "default" in keywords:
default_val = ast.unparse(keywords["default"])
# Env var (auto-generated unless explicitly overridden)
if "env_var" in keywords and isinstance(keywords["env_var"], ast.Constant):
env_var = keywords["env_var"].value
else:
env_var = get_env_var_name(key)
# Type from the class name (e.g. BooleanSetting -> Boolean)
setting_type = node.func.id.replace("Setting", "")
# Required
required = False
if "required" in keywords and isinstance(
keywords["required"], ast.Constant
):
required = bool(keywords["required"].value)
# Min/max value
min_value = None
if "min_value" in keywords and isinstance(
keywords["min_value"], ast.Constant
):
min_value = keywords["min_value"].value
max_value = None
if "max_value" in keywords and isinstance(
keywords["max_value"], ast.Constant
):
max_value = keywords["max_value"].value
# Allowed values (ast.Set of ast.Constant)
allowed_values = None
if "allowed_values" in keywords and isinstance(
keywords["allowed_values"], ast.Set
):
allowed_values = sorted(
elt.value
for elt in keywords["allowed_values"].elts
if isinstance(elt, ast.Constant)
)
# Deprecated env var
deprecated_env_var = None
if "deprecated_env_var" in keywords and isinstance(
keywords["deprecated_env_var"], ast.Constant
):
deprecated_env_var = keywords["deprecated_env_var"].value
return {
"key": key,
"env_var": env_var,
"description": description,
"default": default_val,
"type": setting_type,
"required": required,
"min_value": min_value,
"max_value": max_value,
"allowed_values": allowed_values,
"deprecated_env_var": deprecated_env_var,
}
def get_env_only_settings(
root_dir: Optional[Path] = None,
) -> List[Dict[str, Any]]:
"""
Extract env-only settings from env_definitions/ by auto-discovering modules.
These are settings required before database initialization.
"""
root_dir = root_dir or get_project_root()
env_defs_dir = (
root_dir
/ "src"
/ "local_deep_research"
/ "settings"
/ "env_definitions"
)
env_only: List[Dict[str, Any]] = []
for filepath in _discover_env_definition_files(env_defs_dir):
category = _category_from_filename(filepath.name)
try:
content = filepath.read_text(encoding="utf-8")
tree = ast.parse(content)
except Exception as e:
print(f"Warning: Could not parse {filepath}: {e}")
continue
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
setting = _extract_setting_from_call(node)
if setting is None:
continue
setting["category"] = category
env_only.append(setting)
return env_only
def _format_constraints(setting: Dict[str, Any]) -> str:
"""Build a human-readable constraints string."""
parts = []
if (
setting.get("min_value") is not None
and setting.get("max_value") is not None
):
parts.append(f"{setting['min_value']}..{setting['max_value']}")
elif setting.get("min_value") is not None:
parts.append(f">={setting['min_value']}")
elif setting.get("max_value") is not None:
parts.append(f"<={setting['max_value']}")
if setting.get("allowed_values"):
parts.append(", ".join(setting["allowed_values"]))
return " | ".join(parts) if parts else ""
def generate_docs_content(root_dir: Optional[Path] = None) -> str:
"""Generate the full CONFIGURATION.md content as a string."""
root_dir = root_dir or get_project_root()
defaults_dir = root_dir / "src" / "local_deep_research" / "defaults"
settings: Dict[str, Any] = {}
# Recursively find all JSON files
for json_file in sorted(defaults_dir.rglob("*.json")):
try:
with open(json_file, "r", encoding="utf-8") as f:
data = json.load(f)
settings.update(data)
except Exception as e:
print(f"Warning: Could not load {json_file}: {e}")
sorted_keys = sorted(settings.keys())
# Get env-only settings
env_only_settings = get_env_only_settings(root_dir)
# Build markdown
content = [
"# Configuration Reference",
"",
"This document is automatically generated from the application's default settings.",
"Most settings can be configured via the Web UI (Settings page), or overridden via Environment Variables.",
"",
"## Environment Variables",
"",
"To override a setting using an environment variable, convert the key to uppercase, replace dots with underscores, and prefix with `LDR_`.",
"For example, `app.debug` becomes `LDR_APP_DEBUG`.",
"",
"Configuration Priority: Environment Variables > Web UI Config (Database) > Default Values",
"> Environmental Variables are used to override default values, easing installation, while allowing for adjustments to configuration via Web UI. When an environment variable is set, it takes precedence over any value stored in the database, and the Web UI marks that setting as non-editable. Single-setting API updates and deletes are rejected while the setting is environment-locked; low-level programmatic imports that do not request environment-lock preservation may still be stored but remain shadowed by the environment value on read.",
"",
"`research_library.storage_path` is operator-only. Set `LDR_RESEARCH_LIBRARY_STORAGE_PATH` to change it; existing saved paths stay in use if no override is set, including locations outside `LDR_DATA_DIR`. Audit saved paths when upgrading a multi-user installation. User settings save, delete, import, and reset cannot change this path.",
"",
]
# Env-only section with expanded columns
if env_only_settings:
content.extend(
[
"## Pre-Database (Env-Only) Settings",
"",
"These settings are **required before database initialization** and can only be set via environment variables.",
"They are not available in the Web UI because they are needed to start the application.",
"",
"| Environment Variable | Type | Default | Required | Constraints | Description | Category | Deprecated Alias |",
"|----------------------|------|---------|----------|-------------|-------------|----------|------------------|",
]
)
for setting in sorted(env_only_settings, key=lambda x: x["env_var"]):
env_var = setting["env_var"]
stype = setting["type"]
default = setting["default"]
required = "Yes" if setting.get("required") else "No"
constraints = _format_constraints(setting).replace("|", "\\|")
desc = setting["description"].replace("|", "\\|").replace("\n", " ")
category = setting["category"]
deprecated = setting.get("deprecated_env_var") or ""
row = (
f"| `{env_var}` | {stype} | `{default}` | {required} "
f"| {constraints} | {desc} | {category} | {deprecated} |"
)
content.append(row)
content.extend(["", ""])
# Main settings list
content.extend(
[
"## Settings List",
"",
"| Key | Environment Variable | Default Value | Description | Type |",
"|-----|----------------------|---------------|-------------|------|",
]
)
for key in sorted_keys:
setting = settings[key]
env_var = setting.get("env_var") or get_env_var_name(key)
default_val = format_value(setting.get("value"))
description = (
setting.get("description", "")
.replace("\n", " ")
.replace("|", "\\|")
)
setting_type = setting.get("type", "UNKNOWN")
row = f"| `{key}` | `{env_var}` | `{default_val}` | {description} | {setting_type} |"
content.append(row)
content.append("")
content.append("*Generated by scripts/generate_config_docs.py*")
return "\n".join(content) + "\n"
def generate_docs(
output_path: Optional[Path] = None,
check: bool = False,
) -> int:
"""Generate (or check) CONFIGURATION.md.
Returns 0 on success, 1 if check finds stale docs.
"""
root_dir = get_project_root()
output_file = output_path or (root_dir / "docs" / "CONFIGURATION.md")
new_content = generate_docs_content(root_dir)
if check:
if not output_file.exists():
print(
f"FAIL: {output_file} does not exist. "
"Run 'python scripts/generate_config_docs.py' to generate it."
)
return 1
existing = output_file.read_text(encoding="utf-8")
if existing == new_content:
print("OK: Configuration docs are up to date.")
return 0
print(
f"FAIL: {output_file} is out of date. "
"Run 'python scripts/generate_config_docs.py' to regenerate it."
)
return 1
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_text(new_content, encoding="utf-8")
print(f"Wrote {output_file}")
return 0
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Generate CONFIGURATION.md from defaults"
)
parser.add_argument(
"--output",
"-o",
type=Path,
help="Output file path (default: docs/CONFIGURATION.md)",
)
parser.add_argument(
"--check",
action="store_true",
help="Check if docs are up to date (exit 1 if stale)",
)
args = parser.parse_args()
sys.exit(generate_docs(output_path=args.output, check=args.check))