OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Switch client to admin client to restricted operations (#1772)
Edwin Josee0cfc8a8a1fa
diff --git a/src/api/langflow_ingest.py b/src/api/langflow_ingest.pyindex 186249532..a97d3ec76 100644--- a/src/api/langflow_ingest.py+++ b/src/api/langflow_ingest.py@@ -9,6 +9,7 @@ from dependencies import get_document_index_writer, get_langflow_ingest_token_sefrom services.document_index_writer import DocumentIndexChunk, DocumentIndexWriterfrom services.langflow_ingest_token_service import LangflowIngestTokenServicefrom utils.logging_config import get_logger+from utils.opensearch_utils import opensearch_error_fields, opensearch_error_reasonlogger = get_logger(__name__)@@ -48,6 +49,26 @@ def _authorized_chunk_id(body: LangflowIngestBatch, document_id: str, index: intreturn f"{document_id}_{body.batch_id}_{index}"+def _describe_ingest_error(error: Exception) -> tuple[dict[str, Any], str]:+ """Pull structured OpenSearch context out of an ingest failure.++ OpenSearch transport errors carry the real cause (the exception `type`, the+ `reason`, and on mapping/parse failures the offending field) in their `info`+ body. `str(e)` alone frequently hides that, leaving an opaque 500. This+ returns ``(log_fields, detail)`` so the log line and the 500 detail both name+ the actual cause instead of just the exception class.+ """+ error_str = str(error)+ log_fields: dict[str, Any] = {"error_type": type(error).__name__, "error": error_str}+ log_fields.update(opensearch_error_fields(error))++ detail = error_str+ reason = opensearch_error_reason(error)+ if reason:+ detail = f"{error_str} | opensearch: {reason}"+ return log_fields, detail++async def ingest_langflow_chunks(body: LangflowIngestBatch,authorization: str | None = Header(default=None),@@ -77,14 +98,16 @@ async def ingest_langflow_chunks(try:result = await writer.index_chunks(context, chunks, final=body.final)except Exception as e:+ log_fields, detail = _describe_ingest_error(e)logger.error("Langflow ingest callback failed",ingest_run_id=body.ingest_run_id,batch_id=body.batch_id,chunk_count=len(chunks),- error=str(e),+ document_id=context.document_id,+ **log_fields,)- raise HTTPException(status_code=500, detail=str(e)) from e+ raise HTTPException(status_code=500, detail=detail) from eif body.final:token_service.mark_finalized(jti)diff --git a/src/models/processors.py b/src/models/processors.pyindex 6ab764b83..7a7a950ba 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -505,11 +505,13 @@ class DocumentFileProcessor(TaskProcessor):# Delete existing document before uploading new onelogger.info(f"Replacing existing document: {original_filename}")await self.delete_document_by_filename(original_filename, opensearch_client)- # Refresh index to make deletion visible before processing+ # Refresh index to make deletion visible before processing.+ # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,+ # so it must run under the admin/service client, not the user client.from config.settings import get_index_nametry:- await opensearch_client.indices.refresh(index=get_index_name())+ await clients.opensearch.indices.refresh(index=get_index_name())except Exception as refresh_error:logger.warning("Failed to refresh index after delete",@@ -1073,8 +1075,10 @@ class LangflowFileProcessor(TaskProcessor):owner_user_id=self.owner_user_id,)# Refresh index to make deletion visible before processing.+ # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,+ # so it must run under the admin/service client, not the user client.try:- await opensearch_client.indices.refresh(index=get_index_name())+ await clients.opensearch.indices.refresh(index=get_index_name())except Exception as refresh_error:logger.warning("Failed to refresh index after delete",diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.pyindex 95387c523..f3374386f 100644--- a/src/services/document_index_writer.py+++ b/src/services/document_index_writer.py@@ -260,9 +260,16 @@ class DocumentIndexWriter:def _raise_for_bulk_errors(result: Any) -> None:if not isinstance(result, dict) or not result.get("errors"):return+ # Keep only the items that actually failed (a bulk response interleaves+ # successes and failures) and carry their full OpenSearch error body so+ # the cause — e.g. a mapper_parsing_exception naming the offending field+ # — survives in the raised error. The caller logs this once with request+ # context; this helper only raises so the failure isn't logged twice.failures = []- for item in result.get("items", [])[:5]:+ for item in result.get("items", []):action = item.get("index") or item.get("create") or item.get("update") or item+ if not action.get("error"):+ continuefailures.append({"id": action.get("_id"),@@ -270,6 +277,12 @@ class DocumentIndexWriter:"error": action.get("error"),})+ if len(failures) >= 5:+ break+ if not failures:+ # `errors` was set but no item carried an error body (rare/contradictory);+ # fall back to the first few raw items so the message keeps some detail.+ failures = result.get("items", [])[:3]raise RuntimeError(f"OpenSearch bulk indexing failed: {failures}")async def _refresh(self, index_name: str) -> None:diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.pyindex 5475d4202..a87bd1630 100644--- a/src/utils/opensearch_init.py+++ b/src/utils/opensearch_init.py@@ -370,7 +370,11 @@ async def init_index(opensearch_client=None, admin_username: str = None):await configure_alerting_security()except Exception as e:- from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error+ from utils.opensearch_utils import (+ OpenSearchDiskSpaceError,+ is_disk_space_error,+ opensearch_error_fields,+ )# TransportError stringifies to just "TransportError(500, '')" — surface# the failing step plus the OpenSearch status/error/body it carries.@@ -378,11 +382,8 @@ async def init_index(opensearch_client=None, admin_username: str = None):"failed_step": step,"error": str(e),"error_repr": repr(e),+ **opensearch_error_fields(e),}- if hasattr(e, "status_code"):- err_fields["os_status_code"] = getattr(e, "status_code", None)- err_fields["os_error"] = getattr(e, "error", None)- err_fields["os_info"] = getattr(e, "info", None)logger.error("init_index failed", **err_fields)if is_disk_space_error(e):diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.pyindex 46d729965..68390458c 100644--- a/src/utils/opensearch_utils.py+++ b/src/utils/opensearch_utils.py@@ -1,6 +1,7 @@import asyncioimport osimport random+from typing import Anyimport yamlfrom opensearchpy import AsyncOpenSearch@@ -54,6 +55,54 @@ def is_disk_space_error(error: Exception) -> bool:return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)+def opensearch_error_reason(error: Exception) -> str | None:+ """Return the most specific OpenSearch failure reason carried by an exception.++ OpenSearch transport errors stringify to just ``TransportError(500, '')``; the+ real cause (and, on mapping/parse failures, the offending field) lives in the+ ``info`` body under ``error.root_cause[0].reason`` or ``error.reason``. Returns+ ``None`` when the exception carries no such structured reason.+ """+ info = getattr(error, "info", None)+ if not isinstance(info, dict):+ return None+ err = info.get("error")+ if not isinstance(err, dict):+ return None+ root_cause = err.get("root_cause")+ if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):+ if root_cause[0].get("reason"):+ return root_cause[0]["reason"]+ return err.get("reason")+++def opensearch_error_fields(error: Exception) -> dict[str, Any]:+ """Extract structured OpenSearch context from an exception for logging.++ Surfaces ``opensearch_status``/``opensearch_error``/``opensearch_info`` (and+ ``opensearch_root_cause`` when present) so every OpenSearch-touching call site+ logs the same schema instead of an opaque ``str(e)``. Non-OpenSearch+ exceptions yield an empty dict.+ """+ fields: dict[str, Any] = {}+ status = getattr(error, "status_code", None)+ if status is not None:+ fields["opensearch_status"] = status+ os_error = getattr(error, "error", None)+ if os_error is not None:+ fields["opensearch_error"] = os_error+ info = getattr(error, "info", None)+ if info is not None:+ fields["opensearch_info"] = info+ if isinstance(info, dict):+ err = info.get("error")+ if isinstance(err, dict):+ root_cause = err.get("root_cause")+ if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):+ fields["opensearch_root_cause"] = root_cause[0]+ return fields++async def wait_for_opensearch(opensearch_client: AsyncOpenSearch,max_retries: int = 15,diff --git a/tests/unit/test_document_index_writer_bulk_errors.py b/tests/unit/test_document_index_writer_bulk_errors.pynew file mode 100644index 000000000..2e14c11e9--- /dev/null+++ b/tests/unit/test_document_index_writer_bulk_errors.py@@ -0,0 +1,57 @@+"""`_raise_for_bulk_errors` must surface the real per-item failures.++A bulk response interleaves successes and failures; the check must report only+the items that actually failed, and must never degrade to an empty `[]` message+when the `errors` flag is set.+"""++import pytest++from services.document_index_writer import DocumentIndexWriter+++def test_no_error_when_errors_flag_absent():+ # Should not raise.+ DocumentIndexWriter._raise_for_bulk_errors({"errors": False, "items": []})+ DocumentIndexWriter._raise_for_bulk_errors({"took": 1})+++def test_reports_only_failed_items_from_interleaved_response():+ result = {+ "errors": True,+ "items": [+ {"index": {"_id": "ok-1", "status": 201}},+ {+ "index": {+ "_id": "bad-1",+ "status": 400,+ "error": {"type": "mapper_parsing_exception", "reason": "field [created_time]"},+ }+ },+ {"index": {"_id": "ok-2", "status": 200}},+ ],+ }+ with pytest.raises(RuntimeError) as excinfo:+ DocumentIndexWriter._raise_for_bulk_errors(result)++ message = str(excinfo.value)+ assert "bad-1" in message+ assert "created_time" in message+ # Successful items are not reported as failures.+ assert "ok-1" not in message+ assert "ok-2" not in message+++def test_empty_failures_falls_back_to_raw_items_not_empty_list():+ # `errors` set but no item carries an error body (contradictory/rare).+ result = {+ "errors": True,+ "items": [{"index": {"_id": "x", "status": 201}}],+ }+ with pytest.raises(RuntimeError) as excinfo:+ DocumentIndexWriter._raise_for_bulk_errors(result)++ message = str(excinfo.value)+ # Must keep some detail rather than degrade to "... failed: []".+ assert message != "OpenSearch bulk indexing failed: []"+ assert "x" in messagediff --git a/tests/unit/test_ingest_error_surfacing.py b/tests/unit/test_ingest_error_surfacing.pynew file mode 100644index 000000000..b9e612562--- /dev/null+++ b/tests/unit/test_ingest_error_surfacing.py@@ -0,0 +1,62 @@+"""Connector/Langflow ingest failures must surface the real OpenSearch cause.++A bare `str(e)` on an OpenSearch transport error frequently hides the actual+exception `type`, `reason`, and the offending field, leaving an opaque 500.+`_describe_ingest_error` digs those out of the error `info` body so both the log+line and the 500 detail name the cause (e.g. a `mapper_parsing_exception` on+`created_time` that only connector documents trigger).+"""++from api.langflow_ingest import _describe_ingest_error+++class _TransportErrorLike(Exception):+ """Mimics opensearchpy.TransportError's status_code/info attributes."""++ def __init__(self, status_code, info, message):+ super().__init__(message)+ self.status_code = status_code+ self.info = info+ self._message = message++ def __str__(self):+ return self._message+++def test_describe_ingest_error_extracts_opensearch_reason_and_field():+ info = {+ "error": {+ "type": "mapper_parsing_exception",+ "reason": "failed to parse field [created_time] of type [date]",+ "root_cause": [+ {+ "type": "mapper_parsing_exception",+ "reason": "failed to parse field [created_time] of type [date]",+ }+ ],+ }+ }+ err = _TransportErrorLike(400, info, "TransportError(400, 'mapper_parsing_exception')")++ log_fields, detail = _describe_ingest_error(err)++ assert log_fields["error_type"] == "_TransportErrorLike"+ assert log_fields["opensearch_status"] == 400+ assert log_fields["opensearch_info"] == info+ assert log_fields["opensearch_root_cause"]["type"] == "mapper_parsing_exception"+ # The 500 detail now names the offending field instead of just the class.+ assert "created_time" in detail+ assert "opensearch:" in detail+++def test_describe_ingest_error_plain_exception_is_still_described():+ log_fields, detail = _describe_ingest_error(+ ValueError("Cannot index chunks with empty embeddings")+ )++ assert log_fields["error_type"] == "ValueError"+ assert log_fields["error"] == "Cannot index chunks with empty embeddings"+ # No OpenSearch body to mine -> no synthetic status/info, detail unchanged.+ assert "opensearch_status" not in log_fields+ assert "opensearch_info" not in log_fields+ assert detail == "Cannot index chunks with empty embeddings"diff --git a/tests/unit/test_opensearch_error_fields.py b/tests/unit/test_opensearch_error_fields.pynew file mode 100644index 000000000..63d9ee833--- /dev/null+++ b/tests/unit/test_opensearch_error_fields.py@@ -0,0 +1,63 @@+"""Shared extraction of OpenSearch error context (status/error/info/reason).++`opensearch_error_fields` / `opensearch_error_reason` give every+OpenSearch-touching call site one schema for surfacing the real cause of an+otherwise-opaque `TransportError`, instead of each site re-deriving it with+inconsistent field names.+"""++from utils.opensearch_utils import opensearch_error_fields, opensearch_error_reason+++class _TransportErrorLike(Exception):+ def __init__(self, status_code, error, info, message):+ super().__init__(message)+ self.status_code = status_code+ self.error = error+ self.info = info+ self._message = message++ def __str__(self):+ return self._message+++def _mapping_error():+ info = {+ "error": {+ "type": "mapper_parsing_exception",+ "reason": "top-level reason",+ "root_cause": [+ {+ "type": "mapper_parsing_exception",+ "reason": "failed to parse field [created_time] of type [date]",+ }+ ],+ }+ }+ return _TransportErrorLike(400, "mapper_parsing_exception", info, "TransportError(400, ...)")+++def test_fields_surface_status_error_info_and_root_cause():+ fields = opensearch_error_fields(_mapping_error())++ assert fields["opensearch_status"] == 400+ assert fields["opensearch_error"] == "mapper_parsing_exception"+ assert fields["opensearch_root_cause"]["reason"].endswith("[date]")+ assert "error" in fields["opensearch_info"]+++def test_reason_prefers_root_cause_over_top_level():+ assert opensearch_error_reason(_mapping_error()) == (+ "failed to parse field [created_time] of type [date]"+ )+++def test_reason_falls_back_to_top_level_when_no_root_cause():+ err = _TransportErrorLike(400, "x", {"error": {"reason": "only top level"}}, "msg")+ assert opensearch_error_reason(err) == "only top level"+++def test_plain_exception_yields_no_fields_and_no_reason():+ err = ValueError("not an opensearch error")+ assert opensearch_error_fields(err) == {}+ assert opensearch_error_reason(err) is Nonediff --git a/tests/unit/test_processor_refresh_uses_admin_client.py b/tests/unit/test_processor_refresh_uses_admin_client.pynew file mode 100644index 000000000..ad1be9cec--- /dev/null+++ b/tests/unit/test_processor_refresh_uses_admin_client.py@@ -0,0 +1,117 @@+"""Index refresh after a duplicate-replace must run on the admin client.++`indices:admin/refresh` is index-wide and cannot be DLS-scoped, so the+read-only `openrag_user_role` does not grant it. The visibility check and the+delete are still issued as the user (delete mutates via the admin client+internally), but the trailing refresh must run under the admin/service client+(`clients.opensearch`) — otherwise OpenSearch returns a 403+`security_exception` for `[indices:admin/refresh] ... backend_roles=[openrag_user]`+during the onboarding "replace duplicate" path.++Regression guard for both processors that issue this refresh:+`DocumentFileProcessor` and `LangflowFileProcessor`.+"""++from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++from models.processors import DocumentFileProcessor, LangflowFileProcessor+from models.tasks import FileTask, TaskStatus, UploadTask+++def _admin_clients(monkeypatch):+ """Patch the module-level `clients` with a mock admin OpenSearch client."""+ admin_client = AsyncMock()+ admin_client.indices = AsyncMock()+ monkeypatch.setattr(+ "models.processors.clients",+ SimpleNamespace(opensearch=admin_client),+ )+ return admin_client+++async def test_document_processor_refresh_uses_admin_client(monkeypatch, tmp_path):+ admin_client = _admin_clients(monkeypatch)++ user_client = AsyncMock()+ user_client.indices = AsyncMock()+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=user_client)++ document_service = MagicMock()+ document_service.docling_service = MagicMock()+ document_service.session_manager = session_manager++ processor = DocumentFileProcessor(+ document_service=document_service,+ models_service=MagicMock(),+ owner_user_id="user-1",+ jwt_token="Bearer user-token",+ replace_duplicates=True,+ session_manager=session_manager,+ )+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock()+ processor.process_document_standard = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )++ item = tmp_path / "report.pdf"+ item.write_bytes(b"%PDF-1.4 dummy")+ file_task = FileTask(file_path=str(item))+ file_task.filename = "My Report.pdf"+ upload_task = UploadTask(task_id="task-1", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ assert file_task.status == TaskStatus.COMPLETED+ # Refresh ran on the admin client, never on the DLS-scoped user client.+ admin_client.indices.refresh.assert_awaited_once()+ user_client.indices.refresh.assert_not_awaited()+ # The visibility check and delete still run as the user.+ processor.check_filename_exists.assert_awaited_once()+ processor.delete_document_by_filename.assert_awaited_once()+++async def test_langflow_processor_refresh_uses_admin_client(monkeypatch, tmp_path):+ admin_client = _admin_clients(monkeypatch)++ user_client = AsyncMock()+ user_client.indices = AsyncMock()+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=user_client)++ langflow_file_service = MagicMock()+ langflow_file_service.upload_and_ingest_file = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )++ processor = LangflowFileProcessor(+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ owner_user_id="user-1",+ jwt_token="Bearer user-token",+ replace_duplicates=True,+ )+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock()++ # The Langflow path reads the file off disk after the refresh.+ item = tmp_path / "report.pdf"+ item.write_bytes(b"%PDF-1.4 dummy")++ file_task = FileTask(file_path=str(item))+ file_task.filename = "My Report.pdf"+ upload_task = UploadTask(task_id="task-1", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ assert file_task.status == TaskStatus.COMPLETED+ admin_client.indices.refresh.assert_awaited_once()+ user_client.indices.refresh.assert_not_awaited()+ processor.delete_document_by_filename.assert_awaited_once()+ langflow_file_service.upload_and_ingest_file.assert_awaited_once()