OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: resolve OPENSEARCH_INDEX_NAME consistently across config loaders (#2327)
Mike Fortman1e70ce83cfe6
diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 0d6660b7d..583f5920f 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -80,6 +80,41 @@ def is_permitted_index_name(index_name: str) -> bool:return bool(_PERMITTED_INDEX_NAME.match(index_name))+def apply_index_name_env_override(knowledge: dict[str, Any]) -> None:+ """Apply ``OPENSEARCH_INDEX_NAME`` onto a ``knowledge`` config dict in place.++ The index name is a role-gated infra setting (``securityconfig/roles.yml``),+ shared by every workspace on a deployment, not a user preference. This+ override therefore always wins when the env var is set — independent of the+ storage mode or the ``edited`` flag — so the ingest-write path and the+ connector enrichment path can never resolve different index names (issue+ 81583). Mirrors the unconditional ``legacy_embedding_provider_map`` override.++ An env value outside the security role's index patterns is rejected and the+ prior value kept, since applying it would break search/write with a 403.+ """+ from config.settings import get_opensearch_index_name_override++ env_index_name = get_opensearch_index_name_override()+ if not env_index_name:+ return+ if not is_permitted_index_name(env_index_name):+ logger.error(+ f"OPENSEARCH_INDEX_NAME={env_index_name!r} is not permitted by the "+ f"OpenSearch security role (must match one of "+ f"{ALLOWED_INDEX_NAME_PATTERNS}); ignoring and keeping "+ f"{knowledge.get('index_name', 'documents')!r}. "+ "See securityconfig/roles.yml."+ )+ return+ if knowledge.get("index_name") not in (None, env_index_name):+ logger.warning(+ f"Stored index_name {knowledge.get('index_name')!r} overridden by "+ f"OPENSEARCH_INDEX_NAME={env_index_name!r}"+ )+ knowledge["index_name"] = env_index_name++def _validate_config_path(config_file: str | Path) -> Path:"""Validate a config file path against a strict allowlist (anti path-injection)."""value = os.fspath(config_file)@@ -755,6 +790,10 @@ class ConfigManager:error=str(e),)+ # The index name is infra, not a user preference: it must resolve the+ # same way after onboarding marks the config edited (issue 81583).+ apply_index_name_env_override(config_data["knowledge"])+# Skip all environment overrides if config has been manually editedif temp_config and temp_config.edited:logger.debug("Skipping all env overrides - config marked as edited")@@ -844,18 +883,7 @@ class ConfigManager:config_data["knowledge"]["chunk_size"] = int(os.getenv("CHUNK_SIZE"))if os.getenv("CHUNK_OVERLAP"):config_data["knowledge"]["chunk_overlap"] = int(os.getenv("CHUNK_OVERLAP"))- if os.getenv("OPENSEARCH_INDEX_NAME"):- env_index_name = os.getenv("OPENSEARCH_INDEX_NAME")- if is_permitted_index_name(env_index_name):- config_data["knowledge"]["index_name"] = env_index_name- else:- logger.error(- f"OPENSEARCH_INDEX_NAME={env_index_name!r} is not permitted by the "- f"OpenSearch security role (must match one of "- f"{ALLOWED_INDEX_NAME_PATTERNS}); ignoring and keeping "- f"{config_data['knowledge'].get('index_name', 'documents')!r}. "- "See securityconfig/roles.yml."- )+ # OPENSEARCH_INDEX_NAME is applied above, before the edited-flag gate.if os.getenv("OCR_ENABLED"):config_data["knowledge"]["ocr"] = os.getenv("OCR_ENABLED").lower() in ("true",diff --git a/src/config/settings.py b/src/config/settings.pyindex a6026a00e..19cb8054f 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -33,6 +33,15 @@ def get_legacy_embedding_provider_map_json() -> str | None:return os.getenv("OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP")+def get_opensearch_index_name_override() -> str | None:+ """Return the raw ``OPENSEARCH_INDEX_NAME`` env override.++ This is a role-gated infra setting, not a user preference; it is resolved+ into the ``knowledge`` config by ``apply_index_name_env_override``.+ """+ return os.getenv("OPENSEARCH_INDEX_NAME")++# Environment variablesOPENSEARCH_HOST = os.getenv("OPENSEARCH_HOST", "localhost")OPENSEARCH_PORT = get_env_int("OPENSEARCH_PORT", 9200)diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 03ba8e987..c7bfa8de7 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -1,5 +1,7 @@from typing import Any+from opensearchpy.exceptions import NotFoundError+from config.settings import get_index_namefrom utils.file_utils import clean_connector_filenamefrom utils.logging_config import get_logger@@ -207,6 +209,23 @@ class ConnectorService:)logger.debug(f"Updated metadata for document {document.id}")except Exception as e:+ # A missing index means the chunks aren't where this write expects+ # them (e.g. a residual index-name mismatch, issue 81583). The+ # document is already indexed; metadata enrichment is best-effort+ # and re-runs on the next sync, so don't fail the file over it —+ # matching get_synced_file_ids_for_connector / should_update_acl.+ if (+ isinstance(e, NotFoundError)+ and e.status_code == 404+ and e.error == "index_not_found_exception"+ ):+ logger.warning(+ "Skipping connector metadata enrichment — index not found",+ document_id=document.id,+ index=get_index_name(),+ error=str(e),+ )+ returnlogger.error("OpenSearch metadata update failed",document_id=document.id,diff --git a/src/services/workspace_config_service.py b/src/services/workspace_config_service.pyindex 69381ba8c..5936dd97e 100644--- a/src/services/workspace_config_service.py+++ b/src/services/workspace_config_service.py@@ -29,7 +29,11 @@ from typing import Anyfrom sqlalchemy.ext.asyncio import async_sessionmaker-from config.config_manager import ConfigManager, OpenRAGConfig+from config.config_manager import (+ ConfigManager,+ OpenRAGConfig,+ apply_index_name_env_override,+)from config.storage_mode import (db_writes_enabled,file_writes_enabled,@@ -102,7 +106,7 @@ class WorkspaceConfigService:except Exception as exc: # noqa: BLE001if mode == "db":logger.error("DB read failed in db mode — returning defaults", error=str(exc))- return OpenRAGConfig.from_dict({})+ return self._config_from_db_dict({})logger.warning("DB read failed, falling back to yaml (hybrid)",error=str(exc),@@ -114,7 +118,7 @@ class WorkspaceConfigService:# Pre-migration boot in pure-db mode — return defaults.# The boot-time migration (config_yaml_to_db_v1) will# populate the DB on the next request.- return OpenRAGConfig.from_dict({})+ return self._config_from_db_dict({})return self._cm.load_config() # hybrid fallbackmerged: dict[str, Any] = {@@ -124,10 +128,24 @@ class WorkspaceConfigService:"onboarding": rows.get("onboarding", {}),"edited": (rows.get("meta") or {}).get("edited", False),}- config = OpenRAGConfig.from_dict(merged)+ config = self._config_from_db_dict(merged)self._cm._config = configreturn config+ @staticmethod+ def _config_from_db_dict(data: dict[str, Any]) -> OpenRAGConfig:+ """Build an OpenRAGConfig from DB-derived sections.++ The DB-backed loader never runs ``ConfigManager._load_env_overrides``,+ so the ``OPENSEARCH_INDEX_NAME`` infra override — which the yaml loader+ applies unconditionally — has to be re-applied here or the index name+ silently reverts to the dataclass default in db/hybrid mode (issue+ 81583).+ """+ data.setdefault("knowledge", {})+ apply_index_name_env_override(data["knowledge"])+ return OpenRAGConfig.from_dict(data)+async def get_config(self) -> OpenRAGConfig:if self._cm._config is not None:return self._cm._configdiff --git a/tests/unit/config/test_config_manager_env_overrides.py b/tests/unit/config/test_config_manager_env_overrides.pynew file mode 100644index 000000000..56e77892c--- /dev/null+++ b/tests/unit/config/test_config_manager_env_overrides.py@@ -0,0 +1,66 @@+"""OPENSEARCH_INDEX_NAME is a role-gated infra setting, not a user preference.++It must resolve to the same value regardless of the ``edited`` flag or which+config loader ran. These tests pin that contract for the yaml-backed+``ConfigManager`` loader (see ``test_workspace_config_service.py`` for the+DB-backed loader).++Regression for issue 81583: chunks were written to the env-configured index+while the connector enrichment path resolved the stale default ``documents``.+"""++import sys+import tempfile+from pathlib import Path++import pytest+import yaml++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from config.config_manager import ConfigManager # noqa: E402+++def cfg_file():+ with tempfile.TemporaryDirectory() as tmp:+ yield Path(tmp) / "config.yaml"+++def test_env_index_name_applies_even_when_config_marked_edited(monkeypatch, cfg_file):+ """A workspace onboarded through the UI has ``edited: true``; the env+ override for the index name must still win."""+ cfg_file.write_text(yaml.safe_dump({"edited": True, "knowledge": {}}))+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")++ cm = ConfigManager(config_file=str(cfg_file))+ config = cm.load_config()++ assert config.knowledge.index_name == "orag-documents"+++def test_env_index_name_absent_keeps_stored_value(monkeypatch, cfg_file):+ cfg_file.write_text(+ yaml.safe_dump({"edited": True, "knowledge": {"index_name": "team-documents"}})+ )+ monkeypatch.delenv("OPENSEARCH_INDEX_NAME", raising=False)++ cm = ConfigManager(config_file=str(cfg_file))+ config = cm.load_config()++ assert config.knowledge.index_name == "team-documents"+++def test_env_index_name_not_permitted_is_ignored(monkeypatch, cfg_file):+ """A value outside the OpenSearch security role's index patterns is+ rejected, keeping the prior value rather than breaking access."""+ cfg_file.write_text(yaml.safe_dump({"edited": True, "knowledge": {"index_name": "documents"}}))+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "totally-different-name")++ cm = ConfigManager(config_file=str(cfg_file))+ config = cm.load_config()++ assert config.knowledge.index_name == "documents"diff --git a/tests/unit/connectors/test_update_connector_metadata_missing_index.py b/tests/unit/connectors/test_update_connector_metadata_missing_index.pynew file mode 100644index 000000000..281890d32--- /dev/null+++ b/tests/unit/connectors/test_update_connector_metadata_missing_index.py@@ -0,0 +1,119 @@+"""_update_connector_metadata must not fail a connector file when the target+index is missing.++The document chunks are already indexed by the time enrichment runs; writing+source_url / timestamps / ACL onto them is best-effort and is retried on the+next sync. A transient or residual index-name mismatch (issue 81583) should+degrade to "file OK, warning logged", not a FAILED task.++Mirrors the tolerance already in get_synced_file_ids_for_connector+(connectors.py) and should_update_acl (acl_utils.py).+"""++import sys+from datetime import datetime+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest+from opensearchpy.exceptions import NotFoundError, RequestError++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_service():+ from connectors.service import ConnectorService++ service = ConnectorService.__new__(ConnectorService)+ service.session_manager = MagicMock()+ service.index_name = "orag-documents"++ opensearch_client = AsyncMock()+ service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)+ service.clients = MagicMock()+ service.clients.opensearch = opensearch_client+ return service, opensearch_client+++def _make_document():+ from connectors.base import ConnectorDocument, DocumentACL++ return ConnectorDocument(+ id="stable-connector-id",+ filename="report.pdf",+ mimetype="application/pdf",+ content=b"",+ source_url="https://example.com/report.pdf",+ acl=DocumentACL(owner="alice"),+ modified_time=datetime(2026, 5, 7),+ created_time=datetime(2026, 5, 1),+ metadata={},+ )+++def _noop_acl(monkeypatch):+ async def _acl(**_kwargs):+ return {"status": "unchanged"}++ monkeypatch.setattr("utils.acl_utils.update_document_acl", _acl)+++async def test_missing_index_does_not_raise(_noop_acl, monkeypatch):+ service, opensearch_client = _make_service()+ opensearch_client.update_by_query.side_effect = NotFoundError(+ 404, "index_not_found_exception", "no such index [documents]"+ )++ import connectors.service as service_module++ logger_mock = MagicMock()+ monkeypatch.setattr(service_module, "logger", logger_mock)++ # Must return normally rather than propagate — the caller marks the file+ # FAILED on any exception here.+ await service._update_connector_metadata(+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"+ )++ # The missing index is swallowed with a warning, not an error.+ logger_mock.error.assert_not_called()+ logger_mock.warning.assert_called_once()+ _, warning_kwargs = logger_mock.warning.call_args+ assert warning_kwargs["document_id"] == "stable-connector-id"+ assert "index_not_found_exception" in warning_kwargs["error"]+++async def test_other_opensearch_errors_still_raise(_noop_acl):+ service, opensearch_client = _make_service()+ opensearch_client.update_by_query.side_effect = RequestError(+ 400, "mapper_parsing_exception", "bad script"+ )++ with pytest.raises(RequestError):+ await service._update_connector_metadata(+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"+ )+++async def test_non_index_error_mentioning_token_still_raises(_noop_acl):+ """A non-index failure whose message merely contains the+ 'index_not_found_exception' token must not be swallowed — only a real+ 404 NotFoundError with that structured error type is best-effort."""+ service, opensearch_client = _make_service()+ opensearch_client.update_by_query.side_effect = RequestError(+ 400,+ "search_phase_execution_exception",+ "shard failure caused by index_not_found_exception downstream",+ )++ with pytest.raises(RequestError):+ await service._update_connector_metadata(+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"+ )diff --git a/tests/unit/services/test_workspace_config_service.py b/tests/unit/services/test_workspace_config_service.pyindex da17a5a8a..3c4d97a97 100644--- a/tests/unit/services/test_workspace_config_service.py+++ b/tests/unit/services/test_workspace_config_service.py@@ -1,10 +1,8 @@"""WorkspaceConfigService — yaml/DB dual-write contract."""-import osimport sysimport tempfilefrom pathlib import Path-from unittest.mock import MagicMockimport pytestimport pytest_asyncio@@ -46,30 +44,22 @@ def tmp_config_manager():@pytest.mark.asyncio-async def test_load_config_falls_back_to_yaml_when_db_empty(- tmp_config_manager, session_factory-):- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+async def test_load_config_falls_back_to_yaml_when_db_empty(tmp_config_manager, session_factory):+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)config = await svc.load_config()assert isinstance(config, OpenRAGConfig)assert config.edited is False # fresh install@pytest.mark.asyncio-async def test_save_config_writes_to_yaml_and_db(- monkeypatch, tmp_config_manager, session_factory-):+async def test_save_config_writes_to_yaml_and_db(monkeypatch, tmp_config_manager, session_factory):"""Hybrid mode dual-writes to both yaml and the DB.The default mode is ``db`` (DB-only, no yaml), so this test forceshybrid via an env override to exercise the dual-write contract."""monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)config = tmp_config_manager.load_config()config.agent.system_prompt = "Be helpful."@@ -90,12 +80,8 @@ async def test_save_config_writes_to_yaml_and_db(@pytest.mark.asyncio-async def test_is_onboarded_reads_from_db_first(- tmp_config_manager, session_factory-):- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+async def test_is_onboarded_reads_from_db_first(tmp_config_manager, session_factory):+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)# Seed only the DB (no yaml) — DB read should return Trueasync with session_factory() as session:@@ -106,40 +92,28 @@ async def test_is_onboarded_reads_from_db_first(@pytest.mark.asyncio-async def test_is_onboarded_false_when_neither_set(- tmp_config_manager, session_factory-):- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+async def test_is_onboarded_false_when_neither_set(tmp_config_manager, session_factory):+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)assert await svc.is_onboarded() is False@pytest.mark.asyncio-async def test_get_onboarding_step_returns_db_value(- tmp_config_manager, session_factory-):- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+async def test_get_onboarding_step_returns_db_value(tmp_config_manager, session_factory):+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)async with session_factory() as session:- await WorkspaceConfigRepo(session).upsert(- "onboarding", {"current_step": "agent_setup"}- )+ await WorkspaceConfigRepo(session).upsert("onboarding", {"current_step": "agent_setup"})await session.commit()assert await svc.get_onboarding_step() == "agent_setup"@pytest.mark.asyncio-async def test_yaml_write_hooks_mirror_to_db(- tmp_config_manager, session_factory-):+async def test_yaml_write_hooks_mirror_to_db(tmp_config_manager, session_factory):"""Legacy callers that hit config_manager.save_config_file directlyshould auto-mirror to the DB via the installed monkey-patch."""- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+ # Constructed only for its side effect: the constructor installs the yaml+ # write hooks on tmp_config_manager that mirror direct saves to the DB.+ WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)# Direct legacy-style call (no service, no await)config = tmp_config_manager.load_config()config.agent.llm_model = "gpt-4o"@@ -147,6 +121,7 @@ async def test_yaml_write_hooks_mirror_to_db(# The hook scheduled an asyncio task — wait for it brieflyimport asyncio+for _ in range(20):async with session_factory() as session:agent = await WorkspaceConfigRepo(session).get_section("agent")@@ -164,9 +139,7 @@ async def test_yaml_write_hooks_mirror_to_db(@pytest.mark.asyncioasync def test_kill_switch_disables_db(monkeypatch, tmp_config_manager, session_factory):monkeypatch.setenv("OPENRAG_DISABLE_DB_WORKSPACE_CONFIG", "true")- svc = WorkspaceConfigService(- config_manager=tmp_config_manager, session_factory=session_factory- )+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)config = tmp_config_manager.load_config()config.agent.llm_model = "gpt-5"ok = await svc.save_config(config)@@ -219,3 +192,60 @@ async def test_reinstall_does_not_chain_closures(tmp_config_manager, session_facawait svc1.await_pending_mirrors()await svc2.await_pending_mirrors()delattr(tmp_config_manager, "_db_mirror_installed")+++# ---------------------------------------------------------------------------+# OPENSEARCH_INDEX_NAME override on the DB-backed loader (issue 81583)+# ---------------------------------------------------------------------------+++async def test_db_load_applies_index_name_env_override_when_row_missing_it(+ monkeypatch, tmp_config_manager, session_factory+):+ """db mode builds config straight from DB rows; the env-configured index+ name must still win so the enrichment path can't resolve a stale default."""+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)+ async with session_factory() as session:+ # A knowledge row that predates index_name being persisted, plus the+ # edited marker a completed onboarding leaves behind.+ await WorkspaceConfigRepo(session).upsert("knowledge", {"chunk_size": 512})+ await WorkspaceConfigRepo(session).upsert("meta", {"edited": True})+ await session.commit()++ config = await svc.load_config()++ assert config.knowledge.index_name == "orag-documents"+++async def test_db_load_index_name_env_override_beats_stored_value(+ monkeypatch, tmp_config_manager, session_factory+):+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)+ async with session_factory() as session:+ await WorkspaceConfigRepo(session).upsert("knowledge", {"index_name": "documents"})+ await session.commit()++ config = await svc.load_config()++ assert config.knowledge.index_name == "orag-documents"+++async def test_db_load_applies_index_name_env_override_on_empty_db(+ monkeypatch, tmp_config_manager, session_factory+):+ """Pre-migration boot in pure-db mode returns defaults from an empty DB;+ the index name must come from the env, not the dataclass default."""+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)++ config = await svc.load_config()++ assert config.knowledge.index_name == "orag-documents"