OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: issue #1210 — Langflow ingestion failing with `AuthorizationException(403, 'security_exception', 'no permissions for [indices:data/read/search] ...')`. (#2022)
Richard Zhang4d1be69768b5
diff --git a/.env.example b/.env.exampleindex dc57a733d..6716c271c 100644--- a/.env.example+++ b/.env.example@@ -125,7 +125,10 @@ OPENSEARCH_NODE_COUNT_CHECK_ENABLED=false# OpenSearch index name for storing documents# Default: documents-# Change this if you want to use a different index name or avoid conflicts+# Change this if you want to use a different index name or avoid conflicts.+# Must start with "documents" or "knowledge_filters" — the OpenSearch security+# role (securityconfig/roles.yml) only grants search access to indices+# matching those patterns; any other name causes a 403 AuthorizationException.OPENSEARCH_INDEX_NAME=documents# OpenSearch index layout for newly-created OpenRAG indicesdiff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.pyindex 5be58ec38..01036e5a9 100644--- a/src/api/settings/endpoints.py+++ b/src/api/settings/endpoints.py@@ -7,6 +7,7 @@ Langflow-sync helpers in `api.settings.langflow_sync`. No behavior change."""import asyncio+import copyimport jsonfrom fastapi import Depends, HTTPException, Request@@ -55,6 +56,7 @@ from api.settings.models import (SettingsUpdateResponse,WatsonXProviderConfig,)+from config.config_manager import ALLOWED_INDEX_NAME_PREFIXES, is_permitted_index_namefrom config.settings import (DEFAULT_DOCS_URL,ENVIRONMENT,@@ -391,13 +393,17 @@ async def update_settings(return JSONResponse({"error": f"{str(e)}"}, status_code=400)# Update configuration- # Only reached if validation passed or wasn't needed+ # Only reached if validation passed or wasn't needed.+ # Stage every mutation on a deep copy so a validation failure partway+ # through (e.g. the index_name or chunk_overlap checks below) can't+ # leave the live cached config half-updated and unsaved.+ working_config = copy.deepcopy(current_config)config_updated = False# Update agent settingsif body.llm_model is not None:- old_model = current_config.agent.llm_model- current_config.agent.llm_model = body.llm_model+ old_model = working_config.agent.llm_model+ working_config.agent.llm_model = body.llm_modelconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_LLM_MODEL@@ -405,8 +411,8 @@ async def update_settings(logger.info(f"LLM model changed from {old_model} to {body.llm_model}")if body.llm_provider is not None:- old_provider = current_config.agent.llm_provider- current_config.agent.llm_provider = body.llm_provider+ old_provider = working_config.agent.llm_provider+ working_config.agent.llm_provider = body.llm_providerconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_LLM_PROVIDER@@ -414,7 +420,7 @@ async def update_settings(logger.info(f"LLM provider changed from {old_provider} to {body.llm_provider}")if body.system_prompt is not None:- current_config.agent.system_prompt = body.system_prompt+ working_config.agent.system_prompt = body.system_promptconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_SYSTEM_PROMPT@@ -423,7 +429,7 @@ async def update_settings(# Also update the chat flow with the new system prompttry:flows_service = _get_flows_service()- await _update_langflow_system_prompt(current_config, flows_service)+ await _update_langflow_system_prompt(working_config, flows_service)except Exception as e:logger.error(f"Failed to update chat flow system prompt: {str(e)}")# Don't fail the entire settings update if flow update fails@@ -431,9 +437,9 @@ async def update_settings(# Update knowledge settingsif body.embedding_model is not None:- old_model = current_config.knowledge.embedding_model+ old_model = working_config.knowledge.embedding_modelnew_embedding_model = body.embedding_model.strip()- current_config.knowledge.embedding_model = new_embedding_model+ working_config.knowledge.embedding_model = new_embedding_modelconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_EMBED_MODEL@@ -441,8 +447,8 @@ async def update_settings(logger.info(f"Embedding model changed from {old_model} to {new_embedding_model}")if body.embedding_provider is not None:- old_provider = current_config.knowledge.embedding_provider- current_config.knowledge.embedding_provider = body.embedding_provider+ old_provider = working_config.knowledge.embedding_provider+ working_config.knowledge.embedding_provider = body.embedding_providerconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_EMBED_PROVIDER@@ -452,7 +458,7 @@ async def update_settings()if body.table_structure is not None:- current_config.knowledge.table_structure = body.table_structure+ working_config.knowledge.table_structure = body.table_structureconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_DOCLING_UPDATED@@ -461,12 +467,12 @@ async def update_settings(# Also update the flow with the new docling settingstry:flows_service = _get_flows_service()- await _update_langflow_docling_settings(current_config, flows_service)+ await _update_langflow_docling_settings(working_config, flows_service)except Exception as e:logger.error(f"Failed to update docling settings in flow: {str(e)}")if body.ocr is not None:- current_config.knowledge.ocr = body.ocr+ working_config.knowledge.ocr = body.ocrconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_DOCLING_UPDATED@@ -475,12 +481,12 @@ async def update_settings(# Also update the flow with the new docling settingstry:flows_service = _get_flows_service()- await _update_langflow_docling_settings(current_config, flows_service)+ await _update_langflow_docling_settings(working_config, flows_service)except Exception as e:logger.error(f"Failed to update docling settings in flow: {str(e)}")if body.picture_descriptions is not None:- current_config.knowledge.picture_descriptions = body.picture_descriptions+ working_config.knowledge.picture_descriptions = body.picture_descriptionsconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_DOCLING_UPDATED@@ -489,12 +495,12 @@ async def update_settings(# Also update the flow with the new docling settingstry:flows_service = _get_flows_service()- await _update_langflow_docling_settings(current_config, flows_service)+ await _update_langflow_docling_settings(working_config, flows_service)except Exception as e:logger.error(f"Failed to update docling settings in flow: {str(e)}")if body.disable_ingest_with_langflow is not None:- current_config.knowledge.disable_ingest_with_langflow = (+ working_config.knowledge.disable_ingest_with_langflow = (body.disable_ingest_with_langflow)config_updated = True@@ -506,13 +512,13 @@ async def update_settings(effective_overlap = (body.chunk_overlapif body.chunk_overlap is not None- else current_config.knowledge.chunk_overlap+ else working_config.knowledge.chunk_overlap)if effective_overlap >= body.chunk_size:raise HTTPException(status_code=422, detail="chunk_overlap must be less than chunk_size")- current_config.knowledge.chunk_size = body.chunk_size+ working_config.knowledge.chunk_size = body.chunk_sizeconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_CHUNK_UPDATED@@ -532,13 +538,13 @@ async def update_settings(effective_chunk_size = (body.chunk_sizeif body.chunk_size is not None- else current_config.knowledge.chunk_size+ else working_config.knowledge.chunk_size)if body.chunk_overlap >= effective_chunk_size:raise HTTPException(status_code=422, detail="chunk_overlap must be less than chunk_size")- current_config.knowledge.chunk_overlap = body.chunk_overlap+ working_config.knowledge.chunk_overlap = body.chunk_overlapconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_CHUNK_UPDATED@@ -555,9 +561,18 @@ async def update_settings(logger.error(f"Failed to update ingest flow chunk overlap: {str(e)}")# Don't fail the entire settings update if flow update failsif body.index_name is not None:- old_index_name = current_config.knowledge.index_name+ old_index_name = working_config.knowledge.index_namenew_index_name = body.index_name.strip()- current_config.knowledge.index_name = new_index_name+ if not is_permitted_index_name(new_index_name):+ raise HTTPException(+ status_code=422,+ detail=(+ f"Index name '{new_index_name}' is not permitted. The OpenSearch "+ "security role only grants search access to indices starting "+ f"with {' or '.join(ALLOWED_INDEX_NAME_PREFIXES)}."+ ),+ )+ working_config.knowledge.index_name = new_index_nameconfig_updated = Trueawait TelemetryClient.send_event(Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_INDEX_NAME_UPDATED@@ -581,46 +596,46 @@ async def update_settings(# Update provider-specific settingsprovider_updated = Falseif body.openai_api_key is not None and body.openai_api_key.strip():- current_config.providers.openai.api_key = body.openai_api_key.strip()- current_config.providers.openai.configured = True+ working_config.providers.openai.api_key = body.openai_api_key.strip()+ working_config.providers.openai.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.anthropic_api_key is not None and body.anthropic_api_key.strip():- current_config.providers.anthropic.api_key = body.anthropic_api_key- current_config.providers.anthropic.configured = True+ working_config.providers.anthropic.api_key = body.anthropic_api_key.strip()+ working_config.providers.anthropic.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.watsonx_api_key is not None and body.watsonx_api_key.strip():- current_config.providers.watsonx.api_key = body.watsonx_api_key- current_config.providers.watsonx.configured = True+ working_config.providers.watsonx.api_key = body.watsonx_api_key.strip()+ working_config.providers.watsonx.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.watsonx_endpoint is not None:- current_config.providers.watsonx.endpoint = body.watsonx_endpoint.strip()- current_config.providers.watsonx.configured = True+ working_config.providers.watsonx.endpoint = body.watsonx_endpoint.strip()+ working_config.providers.watsonx.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.watsonx_project_id is not None:- current_config.providers.watsonx.project_id = body.watsonx_project_id.strip()- current_config.providers.watsonx.configured = True+ working_config.providers.watsonx.project_id = body.watsonx_project_id.strip()+ working_config.providers.watsonx.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.ollama_endpoint is not None:- current_config.providers.ollama.endpoint = body.ollama_endpoint.strip()- current_config.providers.ollama.configured = True+ working_config.providers.ollama.endpoint = body.ollama_endpoint.strip()+ working_config.providers.ollama.configured = Trueconfig_updated = Trueprovider_updated = Trueif body.remove_ollama_config:other_providers_configured = (- current_config.providers.openai.configured- or current_config.providers.anthropic.configured- or current_config.providers.watsonx.configured+ working_config.providers.openai.configured+ or working_config.providers.anthropic.configured+ or working_config.providers.watsonx.configured)if not other_providers_configured:return JSONResponse(@@ -635,24 +650,24 @@ async def update_settings()if affected:return _embedding_conflict_response("Ollama", "ollama", affected)- current_config.providers.ollama.endpoint = ""- current_config.providers.ollama.configured = False- if current_config.agent.llm_provider == "ollama":- fb = _first_configured_llm_provider(current_config, "ollama")- current_config.agent.llm_provider = fb- current_config.agent.llm_model = _default_llm_model(fb)- if current_config.knowledge.embedding_provider == "ollama":- fb = _first_configured_embedding_provider(current_config, "ollama")- current_config.knowledge.embedding_provider = fb- current_config.knowledge.embedding_model = _default_embedding_model(fb)+ working_config.providers.ollama.endpoint = ""+ working_config.providers.ollama.configured = False+ if working_config.agent.llm_provider == "ollama":+ fb = _first_configured_llm_provider(working_config, "ollama")+ working_config.agent.llm_provider = fb+ working_config.agent.llm_model = _default_llm_model(fb)+ if working_config.knowledge.embedding_provider == "ollama":+ fb = _first_configured_embedding_provider(working_config, "ollama")+ working_config.knowledge.embedding_provider = fb+ working_config.knowledge.embedding_model = _default_embedding_model(fb)config_updated = Trueprovider_updated = Trueif body.remove_openai_config:other_providers_configured = (- current_config.providers.anthropic.configured- or current_config.providers.watsonx.configured- or current_config.providers.ollama.configured+ working_config.providers.anthropic.configured+ or working_config.providers.watsonx.configured+ or working_config.providers.ollama.configured)if not other_providers_configured:return JSONResponse(@@ -667,24 +682,24 @@ async def update_settings()if affected:return _embedding_conflict_response("OpenAI", "openai", affected)- current_config.providers.openai.api_key = ""- current_config.providers.openai.configured = False- if current_config.agent.llm_provider == "openai":- fb = _first_configured_llm_provider(current_config, "openai")- current_config.agent.llm_provider = fb- current_config.agent.llm_model = _default_llm_model(fb)- if current_config.knowledge.embedding_provider == "openai":- fb = _first_configured_embedding_provider(current_config, "openai")- current_config.knowledge.embedding_provider = fb- current_config.knowledge.embedding_model = _default_embedding_model(fb)+ working_config.providers.openai.api_key = ""+ working_config.providers.openai.configured = False+ if working_config.agent.llm_provider == "openai":+ fb = _first_configured_llm_provider(working_config, "openai")+ working_config.agent.llm_provider = fb+ working_config.agent.llm_model = _default_llm_model(fb)+ if working_config.knowledge.embedding_provider == "openai":+ fb = _first_configured_embedding_provider(working_config, "openai")+ working_config.knowledge.embedding_provider = fb+ working_config.knowledge.embedding_model = _default_embedding_model(fb)config_updated = Trueprovider_updated = Trueif body.remove_anthropic_config:other_providers_configured = (- current_config.providers.openai.configured- or current_config.providers.watsonx.configured- or current_config.providers.ollama.configured+ working_config.providers.openai.configured+ or working_config.providers.watsonx.configured+ or working_config.providers.ollama.configured)if not other_providers_configured:return JSONResponse(@@ -693,21 +708,21 @@ async def update_settings(},status_code=400,)- current_config.providers.anthropic.api_key = ""- current_config.providers.anthropic.configured = False- if current_config.agent.llm_provider == "anthropic":- fb = _first_configured_llm_provider(current_config, "anthropic")- current_config.agent.llm_provider = fb- current_config.agent.llm_model = _default_llm_model(fb)+ working_config.providers.anthropic.api_key = ""+ working_config.providers.anthropic.configured = False+ if working_config.agent.llm_provider == "anthropic":+ fb = _first_configured_llm_provider(working_config, "anthropic")+ working_config.agent.llm_provider = fb+ working_config.agent.llm_model = _default_llm_model(fb)# Anthropic is not a valid embedding provider; no embedding reset neededconfig_updated = Trueprovider_updated = Trueif body.remove_watsonx_config:other_providers_configured = (- current_config.providers.openai.configured- or current_config.providers.anthropic.configured- or current_config.providers.ollama.configured+ working_config.providers.openai.configured+ or working_config.providers.anthropic.configured+ or working_config.providers.ollama.configured)if not other_providers_configured:return JSONResponse(@@ -722,18 +737,18 @@ async def update_settings()if affected:return _embedding_conflict_response("IBM watsonx.ai", "watsonx", affected)- current_config.providers.watsonx.api_key = ""- current_config.providers.watsonx.endpoint = ""- current_config.providers.watsonx.project_id = ""- current_config.providers.watsonx.configured = False- if current_config.agent.llm_provider == "watsonx":- fb = _first_configured_llm_provider(current_config, "watsonx")- current_config.agent.llm_provider = fb- current_config.agent.llm_model = _default_llm_model(fb)- if current_config.knowledge.embedding_provider == "watsonx":- fb = _first_configured_embedding_provider(current_config, "watsonx")- current_config.knowledge.embedding_provider = fb- current_config.knowledge.embedding_model = _default_embedding_model(fb)+ working_config.providers.watsonx.api_key = ""+ working_config.providers.watsonx.endpoint = ""+ working_config.providers.watsonx.project_id = ""+ working_config.providers.watsonx.configured = False+ if working_config.agent.llm_provider == "watsonx":+ fb = _first_configured_llm_provider(working_config, "watsonx")+ working_config.agent.llm_provider = fb+ working_config.agent.llm_model = _default_llm_model(fb)+ if working_config.knowledge.embedding_provider == "watsonx":+ fb = _first_configured_embedding_provider(working_config, "watsonx")+ working_config.knowledge.embedding_provider = fb+ working_config.knowledge.embedding_model = _default_embedding_model(fb)config_updated = Trueprovider_updated = True@@ -746,7 +761,7 @@ async def update_settings(return JSONResponse({"error": "No valid fields provided for update"}, status_code=400)# Save the updated configuration- if not config_manager.save_config_file(current_config):+ if not config_manager.save_config_file(working_config):return JSONResponse({"error": "Failed to save configuration"}, status_code=500)provider_health_cache.invalidate()@@ -785,6 +800,10 @@ async def update_settings()return SettingsUpdateResponse(message="Configuration updated successfully")+ except HTTPException:+ # Preserve intentional client-error status codes (e.g. 422 validation+ # failures) instead of letting the catch-all below flatten them to 500.+ raiseexcept Exception:logger.exception("Failed to update settings")await TelemetryClient.send_event(diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 470c1df52..53b17c5bd 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -58,6 +58,23 @@ logger = get_logger(__name__)# ---------------------------------------------------------------------------_SAFE_CONFIG_PATH = re.compile(r"^/?(?:[A-Za-z0-9_.\-]+/)*[A-Za-z0-9_.\-]+\.ya?ml$")+# ---------------------------------------------------------------------------+# The OpenSearch security role `openrag_user_role` (securityconfig/roles.yml)+# only grants indices:data/read/search on the index_patterns "documents",+# "documents*", "knowledge_filters", "knowledge_filters*". Nothing re-templates+# that static role config from OPENSEARCH_INDEX_NAME, so an index name outside+# those patterns causes ingestion/search to fail with a 403+# AuthorizationException. Keep this allowlist in sync with securityconfig/roles.yml.+# ---------------------------------------------------------------------------+ALLOWED_INDEX_NAME_PREFIXES = ("documents", "knowledge_filters")+_PERMITTED_INDEX_NAME = re.compile(r"^(documents|knowledge_filters)[a-z0-9._-]*$")+++def is_permitted_index_name(index_name: str) -> bool:+ """True if index_name matches an index_pattern the OpenSearch security+ role (securityconfig/roles.yml) actually grants read/search access to."""+ return bool(_PERMITTED_INDEX_NAME.match(index_name))+def _validate_config_path(config_file: str | Path) -> Path:"""Validate a config file path against a strict allowlist (anti path-injection)."""@@ -382,7 +399,17 @@ class ConfigManager:if os.getenv("CHUNK_OVERLAP"):config_data["knowledge"]["chunk_overlap"] = int(os.getenv("CHUNK_OVERLAP"))if os.getenv("OPENSEARCH_INDEX_NAME"):- config_data["knowledge"]["index_name"] = os.getenv("OPENSEARCH_INDEX_NAME")+ env_index_name = os.getenv("OPENSEARCH_INDEX_NAME")+ if is_permitted_index_name(env_index_name):+ config_data["knowledge"]["index_name"] = env_index_name+ else:+ logger.error(+ f"OPENSEARCH_INDEX_NAME={env_index_name!r} is not permitted by the "+ f"OpenSearch security role (must start with one of "+ f"{ALLOWED_INDEX_NAME_PREFIXES}); ignoring and keeping "+ f"{config_data['knowledge'].get('index_name', 'documents')!r}. "+ "See securityconfig/roles.yml."+ )if os.getenv("OCR_ENABLED"):config_data["knowledge"]["ocr"] = os.getenv("OCR_ENABLED").lower() in ("true",diff --git a/tests/unit/test_settings_index_name_validation.py b/tests/unit/test_settings_index_name_validation.pynew file mode 100644index 000000000..e98df0f9f--- /dev/null+++ b/tests/unit/test_settings_index_name_validation.py@@ -0,0 +1,100 @@+from types import SimpleNamespace++import pytest+from fastapi import HTTPException++import api.settings as settings_api+import api.settings.endpoints as settings_endpoints+++def _make_config(index_name="documents"):+ return SimpleNamespace(+ edited=True,+ agent=SimpleNamespace(+ llm_provider="openai", llm_model="gpt-4o", system_prompt="original prompt"+ ),+ knowledge=SimpleNamespace(+ embedding_provider="openai",+ embedding_model="text-embedding-3-small",+ index_name=index_name,+ ),+ providers=SimpleNamespace(),+ )+++async def test_update_settings_rejects_index_name_outside_security_role_patterns(monkeypatch):+ config = _make_config()+ monkeypatch.setattr(settings_endpoints, "get_openrag_config", lambda: config, raising=True)++ with pytest.raises(HTTPException) as exc_info:+ await settings_api.update_settings(+ settings_api.SettingsUpdateBody(index_name="test"),+ session_manager=object(),+ user=None,+ )++ assert exc_info.value.status_code == 422+ # The rejected index name is not permitted, so it must never be written to config.+ assert config.knowledge.index_name == "documents"+++async def test_update_settings_rejects_index_name_without_partially_applying_other_fields(+ monkeypatch,+):+ """A validation failure partway through must not leave earlier fields in+ the same request mutated on the live config object (atomicity)."""+ config = _make_config()+ monkeypatch.setattr(settings_endpoints, "get_openrag_config", lambda: config, raising=True)++ with pytest.raises(HTTPException):+ await settings_api.update_settings(+ settings_api.SettingsUpdateBody(system_prompt="new prompt", index_name="test"),+ session_manager=object(),+ user=None,+ )++ # system_prompt is applied before index_name in the handler, so a pre-fix+ # implementation would have already mutated this field in place.+ assert config.agent.system_prompt == "original prompt"+ assert config.knowledge.index_name == "documents"+++async def test_update_settings_accepts_index_name_matching_security_role_patterns(monkeypatch):+ config = _make_config()+ saved_configs = []+ monkeypatch.setattr(settings_endpoints, "get_openrag_config", lambda: config, raising=True)+ monkeypatch.setattr(+ settings_endpoints.config_manager,+ "save_config_file",+ lambda updated_config: saved_configs.append(updated_config) or True,+ raising=True,+ )++ async def _noop(*args, **kwargs):+ return None++ monkeypatch.setattr(+ settings_endpoints.clients, "_create_langflow_global_variable", _noop, raising=True+ )+ monkeypatch.setattr(settings_endpoints.TelemetryClient, "send_event", _noop, raising=True)+ monkeypatch.setattr(+ settings_endpoints, "_run_async_post_save_langflow_updates", _noop, raising=True+ )+ monkeypatch.setattr(+ settings_endpoints.asyncio, "create_task", lambda coro: coro.close(), raising=True+ )++ await settings_api.update_settings(+ settings_api.SettingsUpdateBody(index_name="documents-v2"),+ session_manager=object(),+ user=None,+ )++ # The staged copy passed to save_config_file carries the new value...+ assert saved_configs[0].knowledge.index_name == "documents-v2"+ # ...but the original config object (the live cache before this call+ # completes) must never be mutated in place.+ assert config.knowledge.index_name == "documents"