OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""DB-backed workspace config — replaces ``config.yaml`` as the source
of truth, with yaml kept as a fallback when storage mode is `hybrid`.
Honors the unified ``OPENRAG_STORAGE_MODE`` flag (see
``src/config/storage_mode.py``):
| Mode | Reads | Writes |
|------------------|-------------------|----------------------|
| db (default) | DB only | DB only — yaml ignored, never written |
| hybrid | DB → yaml fallback| DB + yaml dual-write |
| files | yaml only | yaml only |
Mirrors the existing ``ConfigManager`` API surface (``load_config`` /
``get_config`` / ``reload_config`` / ``save_config_file`` /
``update_onboarding_state``) so call sites in ``src/api/`` don't need
rewrites. Adds two new helpers (``is_onboarded``, ``get_onboarding_step``)
that the new public ``GET /api/onboarding-status`` endpoint uses.
In `hybrid` and `db` modes a one-time monkey-patch on ``config_manager``
intercepts every legacy ``save_config_file`` / ``update_onboarding_state``
call so the DB stays in sync without each call site needing changes.
In `files` mode the patch is not installed.
"""
from __future__ import annotations
import asyncio
from typing import Any
from sqlalchemy.ext.asyncio import async_sessionmaker
from config.config_manager import (
ConfigManager,
OpenRAGConfig,
apply_index_name_env_override,
)
from config.storage_mode import (
db_writes_enabled,
file_writes_enabled,
get_storage_mode,
)
from db.repositories import WorkspaceConfigRepo
from utils.encryption import encrypt_secret
from utils.logging_config import get_logger
logger = get_logger(__name__)
class WorkspaceConfigService:
"""Drop-in replacement for the parts of ConfigManager that touch
persistence. Behavior selected by ``OPENRAG_STORAGE_MODE``."""
def __init__(
self,
config_manager: ConfigManager,
session_factory: async_sessionmaker,
):
self._cm = config_manager
self._session_factory = session_factory
# Track in-flight DB-mirror tasks so the lifespan shutdown can
# await them, and serialize them so rapid double-saves don't
# race on the DB.
self._mirror_lock = asyncio.Lock()
self._pending_mirrors: set[asyncio.Task] = set()
if db_writes_enabled():
self._install_yaml_write_hooks()
logger.info(
"WorkspaceConfigService initialized",
storage_mode=get_storage_mode(),
)
async def await_pending_mirrors(self) -> None:
"""Block until every pending mirror task has finished.
Called from the lifespan shutdown handler so we don't drop
in-flight DB writes when uvicorn cancels remaining tasks.
"""
if not self._pending_mirrors:
return
pending = list(self._pending_mirrors)
logger.info("awaiting pending DB mirror tasks", count=len(pending))
await asyncio.gather(*pending, return_exceptions=True)
# ------------------------------------------------------------------
# Read paths
# ------------------------------------------------------------------
async def load_config(self) -> OpenRAGConfig:
"""Mode-aware read.
- ``files``: yaml only.
- ``db``: DB only — empty DB returns defaults, yaml is ignored.
- ``hybrid``: DB-first, yaml fallback when DB is empty.
Updates the ConfigManager in-process cache so synchronous
``get_openrag_config()`` callers see the same data.
"""
mode = get_storage_mode()
if mode == "files":
return self._cm.load_config()
try:
async with self._session_factory() as session:
repo = WorkspaceConfigRepo(session)
rows = await repo.list_all()
except Exception as exc: # noqa: BLE001
if mode == "db":
logger.error("DB read failed in db mode — returning defaults", error=str(exc))
return self._config_from_db_dict({})
logger.warning(
"DB read failed, falling back to yaml (hybrid)",
error=str(exc),
)
return self._cm.load_config()
if not rows:
if mode == "db":
# Pre-migration boot in pure-db mode — return defaults.
# The boot-time migration (config_yaml_to_db_v1) will
# populate the DB on the next request.
return self._config_from_db_dict({})
return self._cm.load_config() # hybrid fallback
merged: dict[str, Any] = {
"providers": rows.get("providers", {}),
"knowledge": rows.get("knowledge", {}),
"agent": rows.get("agent", {}),
"onboarding": rows.get("onboarding", {}),
"edited": (rows.get("meta") or {}).get("edited", False),
}
config = self._config_from_db_dict(merged)
self._cm._config = config
return config
@staticmethod
def _config_from_db_dict(data: dict[str, Any]) -> OpenRAGConfig:
"""Build an OpenRAGConfig from DB-derived sections.
The DB-backed loader never runs ``ConfigManager._load_env_overrides``,
so the ``OPENSEARCH_INDEX_NAME`` infra override — which the yaml loader
applies unconditionally — has to be re-applied here or the index name
silently reverts to the dataclass default in db/hybrid mode (issue
81583).
"""
data.setdefault("knowledge", {})
apply_index_name_env_override(data["knowledge"])
return OpenRAGConfig.from_dict(data)
async def get_config(self) -> OpenRAGConfig:
if self._cm._config is not None:
return self._cm._config
return await self.load_config()
async def reload_config(self) -> OpenRAGConfig:
self._cm._config = None
return await self.load_config()
async def hydrate_on_startup(self) -> None:
"""Eagerly populate ``config_manager._config`` from the DB at
lifespan startup.
Without this, in `db` mode a restart leaves ``_config = None``
and the synchronous ``get_openrag_config()`` falls back to
defaults (no yaml exists) — so ``/api/settings`` reports
``onboarding.current_step=0`` and the frontend flashes the
wizard on every restart. ``load_config()`` is itself mode-aware,
so this is safe to call unconditionally.
"""
await self.load_config()
async def is_onboarded(self) -> bool:
mode = get_storage_mode()
if mode == "files":
return self._cm.load_config().edited
try:
async with self._session_factory() as session:
repo = WorkspaceConfigRepo(session)
meta = await repo.get_section("meta") or {}
if "edited" in meta:
return bool(meta["edited"])
except Exception as exc: # noqa: BLE001
logger.debug("is_onboarded DB read failed", error=str(exc))
if mode == "db":
return False # no yaml fallback in pure-db mode
return self._cm.load_config().edited
async def get_onboarding_step(self) -> Any | None:
"""Returns the legacy step indicator — usually an int index from
the OnboardingState dataclass, sometimes None. Treat as opaque."""
mode = get_storage_mode()
if mode == "files":
return self._cm.load_config().onboarding.current_step
try:
async with self._session_factory() as session:
repo = WorkspaceConfigRepo(session)
ob = await repo.get_section("onboarding") or {}
if "current_step" in ob:
return ob.get("current_step")
except Exception as exc: # noqa: BLE001
logger.debug("get_onboarding_step DB read failed", error=str(exc))
if mode == "db":
return None
return self._cm.load_config().onboarding.current_step
# ------------------------------------------------------------------
# Write paths
# ------------------------------------------------------------------
async def save_config(
self,
config: OpenRAGConfig | None = None,
*,
preserve_edited: bool = False,
actor_user_id: str | None = None,
) -> bool:
mode = get_storage_mode()
# Yaml write — only when file_writes are enabled.
if file_writes_enabled():
try:
ok = self._cm.save_config_file(config, preserve_edited=preserve_edited)
if not ok:
return False
except Exception as exc: # noqa: BLE001
logger.error("save_config: yaml write failed", error=str(exc))
return False
else:
# In `db` mode we still need to update the in-process
# ConfigManager cache and flip `edited` so callers see the
# new state via the legacy synchronous ``get_openrag_config()``.
self._apply_in_memory(config, preserve_edited=preserve_edited)
if not db_writes_enabled():
return True
try:
await self._mirror_to_db(self._cm.get_config(), actor_user_id=actor_user_id)
except Exception as exc: # noqa: BLE001
if mode == "db":
logger.error("save_config: DB write failed in db mode", error=str(exc))
return False
logger.error("save_config: DB mirror failed (yaml ok)", error=str(exc))
return True
async def update_onboarding_state(
self,
actor_user_id: str | None = None,
**kwargs: Any,
) -> bool:
mode = get_storage_mode()
if file_writes_enabled():
try:
ok = self._cm.update_onboarding_state(**kwargs)
if not ok:
return False
except Exception as exc: # noqa: BLE001
logger.error("update_onboarding_state: yaml failed", error=str(exc))
return False
else:
# In-memory only update for db mode
cfg = self._cm.get_config()
for k, v in kwargs.items():
if hasattr(cfg.onboarding, k):
setattr(cfg.onboarding, k, v)
if not db_writes_enabled():
return True
try:
await self._mirror_to_db(self._cm.get_config(), actor_user_id=actor_user_id)
except Exception as exc: # noqa: BLE001
if mode == "db":
logger.error("update_onboarding_state: DB write failed in db mode", error=str(exc))
return False
logger.error("update_onboarding_state: DB mirror failed", error=str(exc))
return True
# ------------------------------------------------------------------
# Auto-mirror hook (covers legacy callers that bypass this service)
# ------------------------------------------------------------------
def _install_yaml_write_hooks(self) -> None:
"""Wrap ``config_manager.save_config_file`` and
``update_onboarding_state`` so legacy callers in
``src/api/settings.py`` (and elsewhere) auto-mirror to the DB
without per-call-site changes.
In ``db`` mode the patch SKIPS the underlying yaml write but
still updates the in-process cache and schedules the DB mirror —
so ``config_manager`` stays internally consistent without
creating ``config.yaml``.
"""
cm = self._cm
if getattr(cm, "_db_mirror_installed", False):
return
# Pin the *truly original* unpatched methods on the cm BEFORE
# any patching happens. Test cleanup may delete
# `_db_mirror_installed` to force re-install — without this
# capture-once attribute, the second install would close over
# the FIRST install's patched method, building an
# ever-deepening closure chain.
if not hasattr(cm, "_db_mirror_original_save"):
cm._db_mirror_original_save = cm.save_config_file # type: ignore[attr-defined]
cm._db_mirror_original_update_ob = cm.update_onboarding_state # type: ignore[attr-defined]
original_save = cm._db_mirror_original_save # type: ignore[attr-defined]
original_update_ob = cm._db_mirror_original_update_ob # type: ignore[attr-defined]
def patched_save(config=None, preserve_edited: bool = False) -> bool:
mode = get_storage_mode()
if mode == "db":
# Don't write yaml; only update memory + schedule mirror.
self._apply_in_memory(config, preserve_edited=preserve_edited)
self._schedule_mirror()
return True
ok = original_save(config, preserve_edited=preserve_edited)
if ok and db_writes_enabled():
self._schedule_mirror()
return ok
def patched_update_ob(**kwargs) -> bool:
mode = get_storage_mode()
if mode == "db":
cfg = cm.get_config()
for k, v in kwargs.items():
if hasattr(cfg.onboarding, k):
setattr(cfg.onboarding, k, v)
self._schedule_mirror()
return True
ok = original_update_ob(**kwargs)
if ok and db_writes_enabled():
self._schedule_mirror()
return ok
cm.save_config_file = patched_save # type: ignore[method-assign]
cm.update_onboarding_state = patched_update_ob # type: ignore[method-assign]
cm._db_mirror_installed = True # type: ignore[attr-defined]
logger.info(
"WorkspaceConfigService: yaml-write hooks installed",
mode=get_storage_mode(),
)
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
def _apply_in_memory(
self,
config: OpenRAGConfig | None,
preserve_edited: bool,
) -> None:
"""Mirror the in-process effects of ``ConfigManager.save_config_file``
WITHOUT touching disk — used in `db` mode so callers still see
the cache change."""
if config is None:
config = self._cm.get_config()
if not preserve_edited:
config.edited = True
self._cm._config = config
def _schedule_mirror(self) -> None:
"""Fire-and-forget DB mirror of the current config_manager state.
Runs as a tracked asyncio Task on the active loop. Three
guarantees:
1. The config snapshot is captured at SCHEDULE time, not at
task-run time — so rapid `save(A)` then `save(B)` mirrors
*both* states in order, not B twice.
2. An asyncio.Lock serializes mirror writes, preventing two
concurrent ``UPSERT`` against the same row.
3. The task is tracked in ``self._pending_mirrors`` so the
shutdown handler can await it before disposing the engine.
If there's no running loop (sync test context) the mirror is
skipped; the next async-aware caller catches up.
"""
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return
# Snapshot NOW. Reading at task-run time would race with
# subsequent saves to the in-memory config.
config_snapshot = self._cm.get_config()
async def _do_mirror():
try:
async with self._mirror_lock:
await self._mirror_to_db(config_snapshot)
except Exception as exc: # noqa: BLE001
logger.warning("DB mirror after yaml save failed", error=str(exc))
task = loop.create_task(_do_mirror())
self._pending_mirrors.add(task)
task.add_done_callback(self._pending_mirrors.discard)
async def _mirror_to_db(
self,
config: OpenRAGConfig,
*,
actor_user_id: str | None = None,
) -> None:
config_dict = config.to_dict()
providers = dict(config_dict.get("providers", {}))
for prov_name, prov_data in providers.items():
if isinstance(prov_data, dict) and "api_key" in prov_data and prov_data["api_key"]:
prov_data["api_key"] = encrypt_secret(prov_data["api_key"])
providers[prov_name] = prov_data
custom = providers.get("custom", {})
if isinstance(custom, dict):
from services.model_catalog import secret_field_keys
for provider, provider_data in custom.items():
if not isinstance(provider_data, dict):
continue
credentials = provider_data.get("credentials", {})
if not isinstance(credentials, dict):
continue
for key in secret_field_keys(provider):
if credentials.get(key):
credentials[key] = encrypt_secret(credentials[key])
async with self._session_factory() as session:
repo = WorkspaceConfigRepo(session)
# Upsert all non-meta sections wholesale.
for section, value in (
("providers", providers),
("knowledge", config_dict.get("knowledge", {})),
("agent", config_dict.get("agent", {})),
("onboarding", config_dict.get("onboarding", {})),
):
await repo.upsert(section, value, actor_user_id=actor_user_id)
# For "meta", only touch the "edited" flag — preserve every other
# key atomically via a locked merge.
await repo.merge_section_keys(
"meta",
updates={"edited": bool(config_dict.get("edited", False))},
actor_user_id=actor_user_id,
)
await session.commit()