Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: resolve OPENSEARCH_INDEX_NAME consistently across config loaders (#2327)

Mike Fortman1e70ce83cfe6
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 0d6660b7d..583f5920f 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -80,6 +80,41 @@ def is_permitted_index_name(index_name: str) -> bool:
return bool(_PERMITTED_INDEX_NAME.match(index_name))
+def apply_index_name_env_override(knowledge: dict[str, Any]) -> None:
+ """Apply ``OPENSEARCH_INDEX_NAME`` onto a ``knowledge`` config dict in place.
+
+ The index name is a role-gated infra setting (``securityconfig/roles.yml``),
+ shared by every workspace on a deployment, not a user preference. This
+ override therefore always wins when the env var is set — independent of the
+ storage mode or the ``edited`` flag — so the ingest-write path and the
+ connector enrichment path can never resolve different index names (issue
+ 81583). Mirrors the unconditional ``legacy_embedding_provider_map`` override.
+
+ An env value outside the security role's index patterns is rejected and the
+ prior value kept, since applying it would break search/write with a 403.
+ """
+ from config.settings import get_opensearch_index_name_override
+
+ env_index_name = get_opensearch_index_name_override()
+ if not env_index_name:
+ return
+ if not is_permitted_index_name(env_index_name):
+ logger.error(
+ f"OPENSEARCH_INDEX_NAME={env_index_name!r} is not permitted by the "
+ f"OpenSearch security role (must match one of "
+ f"{ALLOWED_INDEX_NAME_PATTERNS}); ignoring and keeping "
+ f"{knowledge.get('index_name', 'documents')!r}. "
+ "See securityconfig/roles.yml."
+ )
+ return
+ if knowledge.get("index_name") not in (None, env_index_name):
+ logger.warning(
+ f"Stored index_name {knowledge.get('index_name')!r} overridden by "
+ f"OPENSEARCH_INDEX_NAME={env_index_name!r}"
+ )
+ knowledge["index_name"] = env_index_name
+
+
def _validate_config_path(config_file: str | Path) -> Path:
"""Validate a config file path against a strict allowlist (anti path-injection)."""
value = os.fspath(config_file)
@@ -755,6 +790,10 @@ class ConfigManager:
error=str(e),
)
+ # The index name is infra, not a user preference: it must resolve the
+ # same way after onboarding marks the config edited (issue 81583).
+ apply_index_name_env_override(config_data["knowledge"])
+
# Skip all environment overrides if config has been manually edited
if temp_config and temp_config.edited:
logger.debug("Skipping all env overrides - config marked as edited")
@@ -844,18 +883,7 @@ class ConfigManager:
config_data["knowledge"]["chunk_size"] = int(os.getenv("CHUNK_SIZE"))
if os.getenv("CHUNK_OVERLAP"):
config_data["knowledge"]["chunk_overlap"] = int(os.getenv("CHUNK_OVERLAP"))
- if os.getenv("OPENSEARCH_INDEX_NAME"):
- env_index_name = os.getenv("OPENSEARCH_INDEX_NAME")
- if is_permitted_index_name(env_index_name):
- config_data["knowledge"]["index_name"] = env_index_name
- else:
- logger.error(
- f"OPENSEARCH_INDEX_NAME={env_index_name!r} is not permitted by the "
- f"OpenSearch security role (must match one of "
- f"{ALLOWED_INDEX_NAME_PATTERNS}); ignoring and keeping "
- f"{config_data['knowledge'].get('index_name', 'documents')!r}. "
- "See securityconfig/roles.yml."
- )
+ # OPENSEARCH_INDEX_NAME is applied above, before the edited-flag gate.
if os.getenv("OCR_ENABLED"):
config_data["knowledge"]["ocr"] = os.getenv("OCR_ENABLED").lower() in (
"true",
diff --git a/src/config/settings.py b/src/config/settings.py
index a6026a00e..19cb8054f 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -33,6 +33,15 @@ def get_legacy_embedding_provider_map_json() -> str | None:
return os.getenv("OPENRAG_LEGACY_EMBEDDING_PROVIDER_MAP")
+def get_opensearch_index_name_override() -> str | None:
+ """Return the raw ``OPENSEARCH_INDEX_NAME`` env override.
+
+ This is a role-gated infra setting, not a user preference; it is resolved
+ into the ``knowledge`` config by ``apply_index_name_env_override``.
+ """
+ return os.getenv("OPENSEARCH_INDEX_NAME")
+
+
# Environment variables
OPENSEARCH_HOST = os.getenv("OPENSEARCH_HOST", "localhost")
OPENSEARCH_PORT = get_env_int("OPENSEARCH_PORT", 9200)
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 03ba8e987..c7bfa8de7 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -1,5 +1,7 @@
from typing import Any
+from opensearchpy.exceptions import NotFoundError
+
from config.settings import get_index_name
from utils.file_utils import clean_connector_filename
from utils.logging_config import get_logger
@@ -207,6 +209,23 @@ class ConnectorService:
)
logger.debug(f"Updated metadata for document {document.id}")
except Exception as e:
+ # A missing index means the chunks aren't where this write expects
+ # them (e.g. a residual index-name mismatch, issue 81583). The
+ # document is already indexed; metadata enrichment is best-effort
+ # and re-runs on the next sync, so don't fail the file over it —
+ # matching get_synced_file_ids_for_connector / should_update_acl.
+ if (
+ isinstance(e, NotFoundError)
+ and e.status_code == 404
+ and e.error == "index_not_found_exception"
+ ):
+ logger.warning(
+ "Skipping connector metadata enrichment — index not found",
+ document_id=document.id,
+ index=get_index_name(),
+ error=str(e),
+ )
+ return
logger.error(
"OpenSearch metadata update failed",
document_id=document.id,
diff --git a/src/services/workspace_config_service.py b/src/services/workspace_config_service.py
index 69381ba8c..5936dd97e 100644
--- a/src/services/workspace_config_service.py
+++ b/src/services/workspace_config_service.py
@@ -29,7 +29,11 @@ from typing import Any
from sqlalchemy.ext.asyncio import async_sessionmaker
-from config.config_manager import ConfigManager, OpenRAGConfig
+from config.config_manager import (
+ ConfigManager,
+ OpenRAGConfig,
+ apply_index_name_env_override,
+)
from config.storage_mode import (
db_writes_enabled,
file_writes_enabled,
@@ -102,7 +106,7 @@ class WorkspaceConfigService:
except Exception as exc: # noqa: BLE001
if mode == "db":
logger.error("DB read failed in db mode — returning defaults", error=str(exc))
- return OpenRAGConfig.from_dict({})
+ return self._config_from_db_dict({})
logger.warning(
"DB read failed, falling back to yaml (hybrid)",
error=str(exc),
@@ -114,7 +118,7 @@ class WorkspaceConfigService:
# Pre-migration boot in pure-db mode — return defaults.
# The boot-time migration (config_yaml_to_db_v1) will
# populate the DB on the next request.
- return OpenRAGConfig.from_dict({})
+ return self._config_from_db_dict({})
return self._cm.load_config() # hybrid fallback
merged: dict[str, Any] = {
@@ -124,10 +128,24 @@ class WorkspaceConfigService:
"onboarding": rows.get("onboarding", {}),
"edited": (rows.get("meta") or {}).get("edited", False),
}
- config = OpenRAGConfig.from_dict(merged)
+ config = self._config_from_db_dict(merged)
self._cm._config = config
return config
+ @staticmethod
+ def _config_from_db_dict(data: dict[str, Any]) -> OpenRAGConfig:
+ """Build an OpenRAGConfig from DB-derived sections.
+
+ The DB-backed loader never runs ``ConfigManager._load_env_overrides``,
+ so the ``OPENSEARCH_INDEX_NAME`` infra override — which the yaml loader
+ applies unconditionally — has to be re-applied here or the index name
+ silently reverts to the dataclass default in db/hybrid mode (issue
+ 81583).
+ """
+ data.setdefault("knowledge", {})
+ apply_index_name_env_override(data["knowledge"])
+ return OpenRAGConfig.from_dict(data)
+
async def get_config(self) -> OpenRAGConfig:
if self._cm._config is not None:
return self._cm._config
diff --git a/tests/unit/config/test_config_manager_env_overrides.py b/tests/unit/config/test_config_manager_env_overrides.py
new file mode 100644
index 000000000..56e77892c
--- /dev/null
+++ b/tests/unit/config/test_config_manager_env_overrides.py
@@ -0,0 +1,66 @@
+"""OPENSEARCH_INDEX_NAME is a role-gated infra setting, not a user preference.
+
+It must resolve to the same value regardless of the ``edited`` flag or which
+config loader ran. These tests pin that contract for the yaml-backed
+``ConfigManager`` loader (see ``test_workspace_config_service.py`` for the
+DB-backed loader).
+
+Regression for issue 81583: chunks were written to the env-configured index
+while the connector enrichment path resolved the stale default ``documents``.
+"""
+
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from config.config_manager import ConfigManager # noqa: E402
+
+
+def cfg_file():
+ with tempfile.TemporaryDirectory() as tmp:
+ yield Path(tmp) / "config.yaml"
+
+
+def test_env_index_name_applies_even_when_config_marked_edited(monkeypatch, cfg_file):
+ """A workspace onboarded through the UI has ``edited: true``; the env
+ override for the index name must still win."""
+ cfg_file.write_text(yaml.safe_dump({"edited": True, "knowledge": {}}))
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")
+
+ cm = ConfigManager(config_file=str(cfg_file))
+ config = cm.load_config()
+
+ assert config.knowledge.index_name == "orag-documents"
+
+
+def test_env_index_name_absent_keeps_stored_value(monkeypatch, cfg_file):
+ cfg_file.write_text(
+ yaml.safe_dump({"edited": True, "knowledge": {"index_name": "team-documents"}})
+ )
+ monkeypatch.delenv("OPENSEARCH_INDEX_NAME", raising=False)
+
+ cm = ConfigManager(config_file=str(cfg_file))
+ config = cm.load_config()
+
+ assert config.knowledge.index_name == "team-documents"
+
+
+def test_env_index_name_not_permitted_is_ignored(monkeypatch, cfg_file):
+ """A value outside the OpenSearch security role's index patterns is
+ rejected, keeping the prior value rather than breaking access."""
+ cfg_file.write_text(yaml.safe_dump({"edited": True, "knowledge": {"index_name": "documents"}}))
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "totally-different-name")
+
+ cm = ConfigManager(config_file=str(cfg_file))
+ config = cm.load_config()
+
+ assert config.knowledge.index_name == "documents"
diff --git a/tests/unit/connectors/test_update_connector_metadata_missing_index.py b/tests/unit/connectors/test_update_connector_metadata_missing_index.py
new file mode 100644
index 000000000..281890d32
--- /dev/null
+++ b/tests/unit/connectors/test_update_connector_metadata_missing_index.py
@@ -0,0 +1,119 @@
+"""_update_connector_metadata must not fail a connector file when the target
+index is missing.
+
+The document chunks are already indexed by the time enrichment runs; writing
+source_url / timestamps / ACL onto them is best-effort and is retried on the
+next sync. A transient or residual index-name mismatch (issue 81583) should
+degrade to "file OK, warning logged", not a FAILED task.
+
+Mirrors the tolerance already in get_synced_file_ids_for_connector
+(connectors.py) and should_update_acl (acl_utils.py).
+"""
+
+import sys
+from datetime import datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from opensearchpy.exceptions import NotFoundError, RequestError
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_service():
+ from connectors.service import ConnectorService
+
+ service = ConnectorService.__new__(ConnectorService)
+ service.session_manager = MagicMock()
+ service.index_name = "orag-documents"
+
+ opensearch_client = AsyncMock()
+ service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ service.clients = MagicMock()
+ service.clients.opensearch = opensearch_client
+ return service, opensearch_client
+
+
+def _make_document():
+ from connectors.base import ConnectorDocument, DocumentACL
+
+ return ConnectorDocument(
+ id="stable-connector-id",
+ filename="report.pdf",
+ mimetype="application/pdf",
+ content=b"",
+ source_url="https://example.com/report.pdf",
+ acl=DocumentACL(owner="alice"),
+ modified_time=datetime(2026, 5, 7),
+ created_time=datetime(2026, 5, 1),
+ metadata={},
+ )
+
+
+def _noop_acl(monkeypatch):
+ async def _acl(**_kwargs):
+ return {"status": "unchanged"}
+
+ monkeypatch.setattr("utils.acl_utils.update_document_acl", _acl)
+
+
+async def test_missing_index_does_not_raise(_noop_acl, monkeypatch):
+ service, opensearch_client = _make_service()
+ opensearch_client.update_by_query.side_effect = NotFoundError(
+ 404, "index_not_found_exception", "no such index [documents]"
+ )
+
+ import connectors.service as service_module
+
+ logger_mock = MagicMock()
+ monkeypatch.setattr(service_module, "logger", logger_mock)
+
+ # Must return normally rather than propagate — the caller marks the file
+ # FAILED on any exception here.
+ await service._update_connector_metadata(
+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"
+ )
+
+ # The missing index is swallowed with a warning, not an error.
+ logger_mock.error.assert_not_called()
+ logger_mock.warning.assert_called_once()
+ _, warning_kwargs = logger_mock.warning.call_args
+ assert warning_kwargs["document_id"] == "stable-connector-id"
+ assert "index_not_found_exception" in warning_kwargs["error"]
+
+
+async def test_other_opensearch_errors_still_raise(_noop_acl):
+ service, opensearch_client = _make_service()
+ opensearch_client.update_by_query.side_effect = RequestError(
+ 400, "mapper_parsing_exception", "bad script"
+ )
+
+ with pytest.raises(RequestError):
+ await service._update_connector_metadata(
+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"
+ )
+
+
+async def test_non_index_error_mentioning_token_still_raises(_noop_acl):
+ """A non-index failure whose message merely contains the
+ 'index_not_found_exception' token must not be swallowed — only a real
+ 404 NotFoundError with that structured error type is best-effort."""
+ service, opensearch_client = _make_service()
+ opensearch_client.update_by_query.side_effect = RequestError(
+ 400,
+ "search_phase_execution_exception",
+ "shard failure caused by index_not_found_exception downstream",
+ )
+
+ with pytest.raises(RequestError):
+ await service._update_connector_metadata(
+ _make_document(), owner_user_id="alice", connector_type="ibm_cos"
+ )
diff --git a/tests/unit/services/test_workspace_config_service.py b/tests/unit/services/test_workspace_config_service.py
index da17a5a8a..3c4d97a97 100644
--- a/tests/unit/services/test_workspace_config_service.py
+++ b/tests/unit/services/test_workspace_config_service.py
@@ -1,10 +1,8 @@
"""WorkspaceConfigService — yaml/DB dual-write contract."""
-import os
import sys
import tempfile
from pathlib import Path
-from unittest.mock import MagicMock
import pytest
import pytest_asyncio
@@ -46,30 +44,22 @@ def tmp_config_manager():
@pytest.mark.asyncio
-async def test_load_config_falls_back_to_yaml_when_db_empty(
- tmp_config_manager, session_factory
-):
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+async def test_load_config_falls_back_to_yaml_when_db_empty(tmp_config_manager, session_factory):
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
config = await svc.load_config()
assert isinstance(config, OpenRAGConfig)
assert config.edited is False # fresh install
@pytest.mark.asyncio
-async def test_save_config_writes_to_yaml_and_db(
- monkeypatch, tmp_config_manager, session_factory
-):
+async def test_save_config_writes_to_yaml_and_db(monkeypatch, tmp_config_manager, session_factory):
"""Hybrid mode dual-writes to both yaml and the DB.
The default mode is ``db`` (DB-only, no yaml), so this test forces
hybrid via an env override to exercise the dual-write contract.
"""
monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
config = tmp_config_manager.load_config()
config.agent.system_prompt = "Be helpful."
@@ -90,12 +80,8 @@ async def test_save_config_writes_to_yaml_and_db(
@pytest.mark.asyncio
-async def test_is_onboarded_reads_from_db_first(
- tmp_config_manager, session_factory
-):
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+async def test_is_onboarded_reads_from_db_first(tmp_config_manager, session_factory):
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
# Seed only the DB (no yaml) — DB read should return True
async with session_factory() as session:
@@ -106,40 +92,28 @@ async def test_is_onboarded_reads_from_db_first(
@pytest.mark.asyncio
-async def test_is_onboarded_false_when_neither_set(
- tmp_config_manager, session_factory
-):
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+async def test_is_onboarded_false_when_neither_set(tmp_config_manager, session_factory):
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
assert await svc.is_onboarded() is False
@pytest.mark.asyncio
-async def test_get_onboarding_step_returns_db_value(
- tmp_config_manager, session_factory
-):
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+async def test_get_onboarding_step_returns_db_value(tmp_config_manager, session_factory):
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
async with session_factory() as session:
- await WorkspaceConfigRepo(session).upsert(
- "onboarding", {"current_step": "agent_setup"}
- )
+ await WorkspaceConfigRepo(session).upsert("onboarding", {"current_step": "agent_setup"})
await session.commit()
assert await svc.get_onboarding_step() == "agent_setup"
@pytest.mark.asyncio
-async def test_yaml_write_hooks_mirror_to_db(
- tmp_config_manager, session_factory
-):
+async def test_yaml_write_hooks_mirror_to_db(tmp_config_manager, session_factory):
"""Legacy callers that hit config_manager.save_config_file directly
should auto-mirror to the DB via the installed monkey-patch."""
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+ # Constructed only for its side effect: the constructor installs the yaml
+ # write hooks on tmp_config_manager that mirror direct saves to the DB.
+ WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
# Direct legacy-style call (no service, no await)
config = tmp_config_manager.load_config()
config.agent.llm_model = "gpt-4o"
@@ -147,6 +121,7 @@ async def test_yaml_write_hooks_mirror_to_db(
# The hook scheduled an asyncio task — wait for it briefly
import asyncio
+
for _ in range(20):
async with session_factory() as session:
agent = await WorkspaceConfigRepo(session).get_section("agent")
@@ -164,9 +139,7 @@ async def test_yaml_write_hooks_mirror_to_db(
@pytest.mark.asyncio
async def test_kill_switch_disables_db(monkeypatch, tmp_config_manager, session_factory):
monkeypatch.setenv("OPENRAG_DISABLE_DB_WORKSPACE_CONFIG", "true")
- svc = WorkspaceConfigService(
- config_manager=tmp_config_manager, session_factory=session_factory
- )
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
config = tmp_config_manager.load_config()
config.agent.llm_model = "gpt-5"
ok = await svc.save_config(config)
@@ -219,3 +192,60 @@ async def test_reinstall_does_not_chain_closures(tmp_config_manager, session_fac
await svc1.await_pending_mirrors()
await svc2.await_pending_mirrors()
delattr(tmp_config_manager, "_db_mirror_installed")
+
+
+# ---------------------------------------------------------------------------
+# OPENSEARCH_INDEX_NAME override on the DB-backed loader (issue 81583)
+# ---------------------------------------------------------------------------
+
+
+async def test_db_load_applies_index_name_env_override_when_row_missing_it(
+ monkeypatch, tmp_config_manager, session_factory
+):
+ """db mode builds config straight from DB rows; the env-configured index
+ name must still win so the enrichment path can't resolve a stale default."""
+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
+ async with session_factory() as session:
+ # A knowledge row that predates index_name being persisted, plus the
+ # edited marker a completed onboarding leaves behind.
+ await WorkspaceConfigRepo(session).upsert("knowledge", {"chunk_size": 512})
+ await WorkspaceConfigRepo(session).upsert("meta", {"edited": True})
+ await session.commit()
+
+ config = await svc.load_config()
+
+ assert config.knowledge.index_name == "orag-documents"
+
+
+async def test_db_load_index_name_env_override_beats_stored_value(
+ monkeypatch, tmp_config_manager, session_factory
+):
+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
+ async with session_factory() as session:
+ await WorkspaceConfigRepo(session).upsert("knowledge", {"index_name": "documents"})
+ await session.commit()
+
+ config = await svc.load_config()
+
+ assert config.knowledge.index_name == "orag-documents"
+
+
+async def test_db_load_applies_index_name_env_override_on_empty_db(
+ monkeypatch, tmp_config_manager, session_factory
+):
+ """Pre-migration boot in pure-db mode returns defaults from an empty DB;
+ the index name must come from the env, not the dataclass default."""
+ monkeypatch.setenv("OPENRAG_STORAGE_MODE", "db")
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", "orag-documents")
+ svc = WorkspaceConfigService(config_manager=tmp_config_manager, session_factory=session_factory)
+
+ config = await svc.load_config()
+
+ assert config.knowledge.index_name == "orag-documents"