Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Switch client to admin client to restricted operations (#1772)

Edwin Josee0cfc8a8a1fa
diff --git a/src/api/langflow_ingest.py b/src/api/langflow_ingest.py
index 186249532..a97d3ec76 100644
--- a/src/api/langflow_ingest.py
+++ b/src/api/langflow_ingest.py
@@ -9,6 +9,7 @@ from dependencies import get_document_index_writer, get_langflow_ingest_token_se
from services.document_index_writer import DocumentIndexChunk, DocumentIndexWriter
from services.langflow_ingest_token_service import LangflowIngestTokenService
from utils.logging_config import get_logger
+from utils.opensearch_utils import opensearch_error_fields, opensearch_error_reason
logger = get_logger(__name__)
@@ -48,6 +49,26 @@ def _authorized_chunk_id(body: LangflowIngestBatch, document_id: str, index: int
return f"{document_id}_{body.batch_id}_{index}"
+def _describe_ingest_error(error: Exception) -> tuple[dict[str, Any], str]:
+ """Pull structured OpenSearch context out of an ingest failure.
+
+ OpenSearch transport errors carry the real cause (the exception `type`, the
+ `reason`, and on mapping/parse failures the offending field) in their `info`
+ body. `str(e)` alone frequently hides that, leaving an opaque 500. This
+ returns ``(log_fields, detail)`` so the log line and the 500 detail both name
+ the actual cause instead of just the exception class.
+ """
+ error_str = str(error)
+ log_fields: dict[str, Any] = {"error_type": type(error).__name__, "error": error_str}
+ log_fields.update(opensearch_error_fields(error))
+
+ detail = error_str
+ reason = opensearch_error_reason(error)
+ if reason:
+ detail = f"{error_str} | opensearch: {reason}"
+ return log_fields, detail
+
+
async def ingest_langflow_chunks(
body: LangflowIngestBatch,
authorization: str | None = Header(default=None),
@@ -77,14 +98,16 @@ async def ingest_langflow_chunks(
try:
result = await writer.index_chunks(context, chunks, final=body.final)
except Exception as e:
+ log_fields, detail = _describe_ingest_error(e)
logger.error(
"Langflow ingest callback failed",
ingest_run_id=body.ingest_run_id,
batch_id=body.batch_id,
chunk_count=len(chunks),
- error=str(e),
+ document_id=context.document_id,
+ **log_fields,
)
- raise HTTPException(status_code=500, detail=str(e)) from e
+ raise HTTPException(status_code=500, detail=detail) from e
if body.final:
token_service.mark_finalized(jti)
diff --git a/src/models/processors.py b/src/models/processors.py
index 6ab764b83..7a7a950ba 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -505,11 +505,13 @@ class DocumentFileProcessor(TaskProcessor):
# Delete existing document before uploading new one
logger.info(f"Replacing existing document: {original_filename}")
await self.delete_document_by_filename(original_filename, opensearch_client)
- # Refresh index to make deletion visible before processing
+ # Refresh index to make deletion visible before processing.
+ # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,
+ # so it must run under the admin/service client, not the user client.
from config.settings import get_index_name
try:
- await opensearch_client.indices.refresh(index=get_index_name())
+ await clients.opensearch.indices.refresh(index=get_index_name())
except Exception as refresh_error:
logger.warning(
"Failed to refresh index after delete",
@@ -1073,8 +1075,10 @@ class LangflowFileProcessor(TaskProcessor):
owner_user_id=self.owner_user_id,
)
# Refresh index to make deletion visible before processing.
+ # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,
+ # so it must run under the admin/service client, not the user client.
try:
- await opensearch_client.indices.refresh(index=get_index_name())
+ await clients.opensearch.indices.refresh(index=get_index_name())
except Exception as refresh_error:
logger.warning(
"Failed to refresh index after delete",
diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.py
index 95387c523..f3374386f 100644
--- a/src/services/document_index_writer.py
+++ b/src/services/document_index_writer.py
@@ -260,9 +260,16 @@ class DocumentIndexWriter:
def _raise_for_bulk_errors(result: Any) -> None:
if not isinstance(result, dict) or not result.get("errors"):
return
+ # Keep only the items that actually failed (a bulk response interleaves
+ # successes and failures) and carry their full OpenSearch error body so
+ # the cause — e.g. a mapper_parsing_exception naming the offending field
+ # — survives in the raised error. The caller logs this once with request
+ # context; this helper only raises so the failure isn't logged twice.
failures = []
- for item in result.get("items", [])[:5]:
+ for item in result.get("items", []):
action = item.get("index") or item.get("create") or item.get("update") or item
+ if not action.get("error"):
+ continue
failures.append(
{
"id": action.get("_id"),
@@ -270,6 +277,12 @@ class DocumentIndexWriter:
"error": action.get("error"),
}
)
+ if len(failures) >= 5:
+ break
+ if not failures:
+ # `errors` was set but no item carried an error body (rare/contradictory);
+ # fall back to the first few raw items so the message keeps some detail.
+ failures = result.get("items", [])[:3]
raise RuntimeError(f"OpenSearch bulk indexing failed: {failures}")
async def _refresh(self, index_name: str) -> None:
diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.py
index 5475d4202..a87bd1630 100644
--- a/src/utils/opensearch_init.py
+++ b/src/utils/opensearch_init.py
@@ -370,7 +370,11 @@ async def init_index(opensearch_client=None, admin_username: str = None):
await configure_alerting_security()
except Exception as e:
- from utils.opensearch_utils import OpenSearchDiskSpaceError, is_disk_space_error
+ from utils.opensearch_utils import (
+ OpenSearchDiskSpaceError,
+ is_disk_space_error,
+ opensearch_error_fields,
+ )
# TransportError stringifies to just "TransportError(500, '')" — surface
# the failing step plus the OpenSearch status/error/body it carries.
@@ -378,11 +382,8 @@ async def init_index(opensearch_client=None, admin_username: str = None):
"failed_step": step,
"error": str(e),
"error_repr": repr(e),
+ **opensearch_error_fields(e),
}
- if hasattr(e, "status_code"):
- err_fields["os_status_code"] = getattr(e, "status_code", None)
- err_fields["os_error"] = getattr(e, "error", None)
- err_fields["os_info"] = getattr(e, "info", None)
logger.error("init_index failed", **err_fields)
if is_disk_space_error(e):
diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.py
index 46d729965..68390458c 100644
--- a/src/utils/opensearch_utils.py
+++ b/src/utils/opensearch_utils.py
@@ -1,6 +1,7 @@
import asyncio
import os
import random
+from typing import Any
import yaml
from opensearchpy import AsyncOpenSearch
@@ -54,6 +55,54 @@ def is_disk_space_error(error: Exception) -> bool:
return any(indicator in error_str for indicator in _DISK_SPACE_INDICATORS)
+def opensearch_error_reason(error: Exception) -> str | None:
+ """Return the most specific OpenSearch failure reason carried by an exception.
+
+ OpenSearch transport errors stringify to just ``TransportError(500, '')``; the
+ real cause (and, on mapping/parse failures, the offending field) lives in the
+ ``info`` body under ``error.root_cause[0].reason`` or ``error.reason``. Returns
+ ``None`` when the exception carries no such structured reason.
+ """
+ info = getattr(error, "info", None)
+ if not isinstance(info, dict):
+ return None
+ err = info.get("error")
+ if not isinstance(err, dict):
+ return None
+ root_cause = err.get("root_cause")
+ if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):
+ if root_cause[0].get("reason"):
+ return root_cause[0]["reason"]
+ return err.get("reason")
+
+
+def opensearch_error_fields(error: Exception) -> dict[str, Any]:
+ """Extract structured OpenSearch context from an exception for logging.
+
+ Surfaces ``opensearch_status``/``opensearch_error``/``opensearch_info`` (and
+ ``opensearch_root_cause`` when present) so every OpenSearch-touching call site
+ logs the same schema instead of an opaque ``str(e)``. Non-OpenSearch
+ exceptions yield an empty dict.
+ """
+ fields: dict[str, Any] = {}
+ status = getattr(error, "status_code", None)
+ if status is not None:
+ fields["opensearch_status"] = status
+ os_error = getattr(error, "error", None)
+ if os_error is not None:
+ fields["opensearch_error"] = os_error
+ info = getattr(error, "info", None)
+ if info is not None:
+ fields["opensearch_info"] = info
+ if isinstance(info, dict):
+ err = info.get("error")
+ if isinstance(err, dict):
+ root_cause = err.get("root_cause")
+ if isinstance(root_cause, list) and root_cause and isinstance(root_cause[0], dict):
+ fields["opensearch_root_cause"] = root_cause[0]
+ return fields
+
+
async def wait_for_opensearch(
opensearch_client: AsyncOpenSearch,
max_retries: int = 15,
diff --git a/tests/unit/test_document_index_writer_bulk_errors.py b/tests/unit/test_document_index_writer_bulk_errors.py
new file mode 100644
index 000000000..2e14c11e9
--- /dev/null
+++ b/tests/unit/test_document_index_writer_bulk_errors.py
@@ -0,0 +1,57 @@
+"""`_raise_for_bulk_errors` must surface the real per-item failures.
+
+A bulk response interleaves successes and failures; the check must report only
+the items that actually failed, and must never degrade to an empty `[]` message
+when the `errors` flag is set.
+"""
+
+import pytest
+
+from services.document_index_writer import DocumentIndexWriter
+
+
+def test_no_error_when_errors_flag_absent():
+ # Should not raise.
+ DocumentIndexWriter._raise_for_bulk_errors({"errors": False, "items": []})
+ DocumentIndexWriter._raise_for_bulk_errors({"took": 1})
+
+
+def test_reports_only_failed_items_from_interleaved_response():
+ result = {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "ok-1", "status": 201}},
+ {
+ "index": {
+ "_id": "bad-1",
+ "status": 400,
+ "error": {"type": "mapper_parsing_exception", "reason": "field [created_time]"},
+ }
+ },
+ {"index": {"_id": "ok-2", "status": 200}},
+ ],
+ }
+ with pytest.raises(RuntimeError) as excinfo:
+ DocumentIndexWriter._raise_for_bulk_errors(result)
+
+ message = str(excinfo.value)
+ assert "bad-1" in message
+ assert "created_time" in message
+ # Successful items are not reported as failures.
+ assert "ok-1" not in message
+ assert "ok-2" not in message
+
+
+def test_empty_failures_falls_back_to_raw_items_not_empty_list():
+ # `errors` set but no item carries an error body (contradictory/rare).
+ result = {
+ "errors": True,
+ "items": [{"index": {"_id": "x", "status": 201}}],
+ }
+ with pytest.raises(RuntimeError) as excinfo:
+ DocumentIndexWriter._raise_for_bulk_errors(result)
+
+ message = str(excinfo.value)
+ # Must keep some detail rather than degrade to "... failed: []".
+ assert message != "OpenSearch bulk indexing failed: []"
+ assert "x" in message
diff --git a/tests/unit/test_ingest_error_surfacing.py b/tests/unit/test_ingest_error_surfacing.py
new file mode 100644
index 000000000..b9e612562
--- /dev/null
+++ b/tests/unit/test_ingest_error_surfacing.py
@@ -0,0 +1,62 @@
+"""Connector/Langflow ingest failures must surface the real OpenSearch cause.
+
+A bare `str(e)` on an OpenSearch transport error frequently hides the actual
+exception `type`, `reason`, and the offending field, leaving an opaque 500.
+`_describe_ingest_error` digs those out of the error `info` body so both the log
+line and the 500 detail name the cause (e.g. a `mapper_parsing_exception` on
+`created_time` that only connector documents trigger).
+"""
+
+from api.langflow_ingest import _describe_ingest_error
+
+
+class _TransportErrorLike(Exception):
+ """Mimics opensearchpy.TransportError's status_code/info attributes."""
+
+ def __init__(self, status_code, info, message):
+ super().__init__(message)
+ self.status_code = status_code
+ self.info = info
+ self._message = message
+
+ def __str__(self):
+ return self._message
+
+
+def test_describe_ingest_error_extracts_opensearch_reason_and_field():
+ info = {
+ "error": {
+ "type": "mapper_parsing_exception",
+ "reason": "failed to parse field [created_time] of type [date]",
+ "root_cause": [
+ {
+ "type": "mapper_parsing_exception",
+ "reason": "failed to parse field [created_time] of type [date]",
+ }
+ ],
+ }
+ }
+ err = _TransportErrorLike(400, info, "TransportError(400, 'mapper_parsing_exception')")
+
+ log_fields, detail = _describe_ingest_error(err)
+
+ assert log_fields["error_type"] == "_TransportErrorLike"
+ assert log_fields["opensearch_status"] == 400
+ assert log_fields["opensearch_info"] == info
+ assert log_fields["opensearch_root_cause"]["type"] == "mapper_parsing_exception"
+ # The 500 detail now names the offending field instead of just the class.
+ assert "created_time" in detail
+ assert "opensearch:" in detail
+
+
+def test_describe_ingest_error_plain_exception_is_still_described():
+ log_fields, detail = _describe_ingest_error(
+ ValueError("Cannot index chunks with empty embeddings")
+ )
+
+ assert log_fields["error_type"] == "ValueError"
+ assert log_fields["error"] == "Cannot index chunks with empty embeddings"
+ # No OpenSearch body to mine -> no synthetic status/info, detail unchanged.
+ assert "opensearch_status" not in log_fields
+ assert "opensearch_info" not in log_fields
+ assert detail == "Cannot index chunks with empty embeddings"
diff --git a/tests/unit/test_opensearch_error_fields.py b/tests/unit/test_opensearch_error_fields.py
new file mode 100644
index 000000000..63d9ee833
--- /dev/null
+++ b/tests/unit/test_opensearch_error_fields.py
@@ -0,0 +1,63 @@
+"""Shared extraction of OpenSearch error context (status/error/info/reason).
+
+`opensearch_error_fields` / `opensearch_error_reason` give every
+OpenSearch-touching call site one schema for surfacing the real cause of an
+otherwise-opaque `TransportError`, instead of each site re-deriving it with
+inconsistent field names.
+"""
+
+from utils.opensearch_utils import opensearch_error_fields, opensearch_error_reason
+
+
+class _TransportErrorLike(Exception):
+ def __init__(self, status_code, error, info, message):
+ super().__init__(message)
+ self.status_code = status_code
+ self.error = error
+ self.info = info
+ self._message = message
+
+ def __str__(self):
+ return self._message
+
+
+def _mapping_error():
+ info = {
+ "error": {
+ "type": "mapper_parsing_exception",
+ "reason": "top-level reason",
+ "root_cause": [
+ {
+ "type": "mapper_parsing_exception",
+ "reason": "failed to parse field [created_time] of type [date]",
+ }
+ ],
+ }
+ }
+ return _TransportErrorLike(400, "mapper_parsing_exception", info, "TransportError(400, ...)")
+
+
+def test_fields_surface_status_error_info_and_root_cause():
+ fields = opensearch_error_fields(_mapping_error())
+
+ assert fields["opensearch_status"] == 400
+ assert fields["opensearch_error"] == "mapper_parsing_exception"
+ assert fields["opensearch_root_cause"]["reason"].endswith("[date]")
+ assert "error" in fields["opensearch_info"]
+
+
+def test_reason_prefers_root_cause_over_top_level():
+ assert opensearch_error_reason(_mapping_error()) == (
+ "failed to parse field [created_time] of type [date]"
+ )
+
+
+def test_reason_falls_back_to_top_level_when_no_root_cause():
+ err = _TransportErrorLike(400, "x", {"error": {"reason": "only top level"}}, "msg")
+ assert opensearch_error_reason(err) == "only top level"
+
+
+def test_plain_exception_yields_no_fields_and_no_reason():
+ err = ValueError("not an opensearch error")
+ assert opensearch_error_fields(err) == {}
+ assert opensearch_error_reason(err) is None
diff --git a/tests/unit/test_processor_refresh_uses_admin_client.py b/tests/unit/test_processor_refresh_uses_admin_client.py
new file mode 100644
index 000000000..ad1be9cec
--- /dev/null
+++ b/tests/unit/test_processor_refresh_uses_admin_client.py
@@ -0,0 +1,117 @@
+"""Index refresh after a duplicate-replace must run on the admin client.
+
+`indices:admin/refresh` is index-wide and cannot be DLS-scoped, so the
+read-only `openrag_user_role` does not grant it. The visibility check and the
+delete are still issued as the user (delete mutates via the admin client
+internally), but the trailing refresh must run under the admin/service client
+(`clients.opensearch`) — otherwise OpenSearch returns a 403
+`security_exception` for `[indices:admin/refresh] ... backend_roles=[openrag_user]`
+during the onboarding "replace duplicate" path.
+
+Regression guard for both processors that issue this refresh:
+`DocumentFileProcessor` and `LangflowFileProcessor`.
+"""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from models.processors import DocumentFileProcessor, LangflowFileProcessor
+from models.tasks import FileTask, TaskStatus, UploadTask
+
+
+def _admin_clients(monkeypatch):
+ """Patch the module-level `clients` with a mock admin OpenSearch client."""
+ admin_client = AsyncMock()
+ admin_client.indices = AsyncMock()
+ monkeypatch.setattr(
+ "models.processors.clients",
+ SimpleNamespace(opensearch=admin_client),
+ )
+ return admin_client
+
+
+async def test_document_processor_refresh_uses_admin_client(monkeypatch, tmp_path):
+ admin_client = _admin_clients(monkeypatch)
+
+ user_client = AsyncMock()
+ user_client.indices = AsyncMock()
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=user_client)
+
+ document_service = MagicMock()
+ document_service.docling_service = MagicMock()
+ document_service.session_manager = session_manager
+
+ processor = DocumentFileProcessor(
+ document_service=document_service,
+ models_service=MagicMock(),
+ owner_user_id="user-1",
+ jwt_token="Bearer user-token",
+ replace_duplicates=True,
+ session_manager=session_manager,
+ )
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock()
+ processor.process_document_standard = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+
+ item = tmp_path / "report.pdf"
+ item.write_bytes(b"%PDF-1.4 dummy")
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "My Report.pdf"
+ upload_task = UploadTask(task_id="task-1", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ # Refresh ran on the admin client, never on the DLS-scoped user client.
+ admin_client.indices.refresh.assert_awaited_once()
+ user_client.indices.refresh.assert_not_awaited()
+ # The visibility check and delete still run as the user.
+ processor.check_filename_exists.assert_awaited_once()
+ processor.delete_document_by_filename.assert_awaited_once()
+
+
+async def test_langflow_processor_refresh_uses_admin_client(monkeypatch, tmp_path):
+ admin_client = _admin_clients(monkeypatch)
+
+ user_client = AsyncMock()
+ user_client.indices = AsyncMock()
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=user_client)
+
+ langflow_file_service = MagicMock()
+ langflow_file_service.upload_and_ingest_file = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+
+ processor = LangflowFileProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ owner_user_id="user-1",
+ jwt_token="Bearer user-token",
+ replace_duplicates=True,
+ )
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock()
+
+ # The Langflow path reads the file off disk after the refresh.
+ item = tmp_path / "report.pdf"
+ item.write_bytes(b"%PDF-1.4 dummy")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "My Report.pdf"
+ upload_task = UploadTask(task_id="task-1", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ admin_client.indices.refresh.assert_awaited_once()
+ user_client.indices.refresh.assert_not_awaited()
+ processor.delete_document_by_filename.assert_awaited_once()
+ langflow_file_service.upload_and_ingest_file.assert_awaited_once()