Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: post-ingestion verification must not fail files on OpenSearch 0utages (#1868)

Edwin Josec36281000243
diff --git a/src/models/processors.py b/src/models/processors.py
index 744b46b25..dbf885d69 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -2,7 +2,7 @@ import asyncio
import mimetypes
import os
import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
from utils.document_processing import (
@@ -29,6 +29,16 @@ if TYPE_CHECKING:
from connectors.base import DocumentACL
+def _verification_client(fallback_client):
+ """Client for post-ingestion verification ("did the chunks land in the
+ index?"). That is a system integrity check, not a user-visibility check,
+ so prefer the platform writer client: it does not depend on the JWT/JWKS
+ trust chain that user-scoped clients need (OpenSearch loads the backend's
+ JWKS lazily, so the first user-JWT queries after a cold start can 401).
+ Falls back to the caller's client when the writer is unavailable."""
+ return clients.opensearch if clients.opensearch is not None else fallback_client
+
+
class TaskProcessor:
"""Base class for task processors with shared processing logic"""
@@ -41,10 +51,19 @@ class TaskProcessor:
self,
file_hash: str,
opensearch_client,
+ on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",
) -> bool:
"""
Check if a document with the given hash already exists in OpenSearch.
Consolidated hash checking for all processors.
+
+ ``on_error`` picks the answer when OpenSearch stays unreachable after
+ retries — the check is ambiguous then, and the safe default differs by
+ caller:
+ * ``"assume_missing"`` (dedupe callers): safer to reprocess than skip.
+ * ``"assume_exists"`` (post-ingestion verification callers): an
+ infrastructure error must not fail a file that Langflow already
+ reported as ingested.
"""
max_retries = 3
retry_delay = 1.0
@@ -69,7 +88,14 @@ class TaskProcessor:
error=str(e),
attempt=attempt + 1,
)
- # On final failure, assume document doesn't exist (safer to reprocess than skip)
+ if on_error == "assume_exists":
+ logger.warning(
+ "Exists check inconclusive due to connection issues; "
+ "assuming document exists",
+ file_hash=file_hash,
+ )
+ return True
+ # Safer to reprocess than skip for dedupe callers.
logger.warning(
"Assuming document doesn't exist due to connection issues",
file_hash=file_hash,
@@ -85,7 +111,7 @@ class TaskProcessor:
)
await asyncio.sleep(retry_delay)
retry_delay *= 2 # Exponential backoff
- return False
+ return on_error == "assume_exists"
async def check_filename_exists(
self,
@@ -868,7 +894,11 @@ class ConnectorFileProcessor(TaskProcessor):
# Langflow returns "success" even when no text was extracted
# (e.g. image files without OCR). Verify the document actually
# landed in OpenSearch before declaring success.
- if not await self.check_document_exists(document.id, opensearch_client):
+ if not await self.check_document_exists(
+ document.id,
+ _verification_client(opensearch_client),
+ on_error="assume_exists",
+ ):
result = {
"status": "error",
"error": "No text content could be extracted from document",
@@ -1157,7 +1187,11 @@ class LangflowFileProcessor(TaskProcessor):
# Langflow returns "success" even when no text was extracted.
# Verify the document actually landed in OpenSearch.
file_hash = hash_id(item)
- if not await self.check_document_exists(file_hash, opensearch_client):
+ if not await self.check_document_exists(
+ file_hash,
+ _verification_client(opensearch_client),
+ on_error="assume_exists",
+ ):
file_task.status = TaskStatus.FAILED
file_task.error = "No text content could be extracted from document"
file_task.updated_at = time.time()
diff --git a/tests/unit/test_processor_exists_verification.py b/tests/unit/test_processor_exists_verification.py
new file mode 100644
index 000000000..945061ef2
--- /dev/null
+++ b/tests/unit/test_processor_exists_verification.py
@@ -0,0 +1,100 @@
+"""Post-ingestion verification must not fail files on OpenSearch outages.
+
+#1851 added "verify the document actually landed in OpenSearch" checks after
+Langflow ingestion. ``check_document_exists`` returns False on persistent
+OpenSearch errors ("safer to reprocess than skip") — correct for its dedupe
+callers, but the verification callers interpreted that same False as "the
+chunks never landed" and marked files FAILED. A transient auth/connectivity
+blip (e.g. OpenSearch's lazy JWKS load returning 401s right after startup)
+then failed every in-flight file, which is what broke
+test_onboarding_sample_docs in CI.
+
+Covers the ``on_error`` mode on ``check_document_exists`` and the
+``_verification_client`` selection helper.
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+import models.processors as processors_module
+from models.processors import TaskProcessor, _verification_client
+
+
+def _failing_client() -> MagicMock:
+ client = MagicMock()
+ client.search = AsyncMock(
+ side_effect=ConnectionError("AuthenticationException(401, 'Unauthorized')")
+ )
+ return client
+
+
+def _hit_client(has_hit: bool) -> MagicMock:
+ client = MagicMock()
+ client.search = AsyncMock(return_value={"hits": {"hits": [{"_id": "x"}] if has_hit else []}})
+ return client
+
+
[email protected](autouse=True)
+def _fast_retries(monkeypatch):
+ """Skip the exponential-backoff sleeps so retry exhaustion is instant.
+
+ Returns the sleep mock so tests can assert on the retry count."""
+ sleep_mock = AsyncMock()
+ monkeypatch.setattr(processors_module.asyncio, "sleep", sleep_mock)
+ return sleep_mock
+
+
+async def test_exists_check_error_defaults_to_missing():
+ """Dedupe callers keep the historical contract: error -> reprocess."""
+ exists = await TaskProcessor().check_document_exists("hash", _failing_client())
+ assert exists is False
+
+
+async def test_exists_check_error_assume_exists_for_verification():
+ """Verification callers must not turn an infra error into a FAILED file."""
+ exists = await TaskProcessor().check_document_exists(
+ "hash", _failing_client(), on_error="assume_exists"
+ )
+ assert exists is True
+
+
+ ("on_error", "expected"),
+ [("assume_missing", False), ("assume_exists", True)],
+)
+async def test_exists_check_exhausts_retries_before_error_fallback(
+ _fast_retries, on_error, expected
+):
+ """Both modes must still exhaust all retries before giving up — the
+ error fallback only fires on the final attempt."""
+ client = _failing_client()
+ exists = await TaskProcessor().check_document_exists("hash", client, on_error=on_error)
+ assert exists is expected
+ assert client.search.await_count == 3 # all attempts used
+ assert _fast_retries.await_count == 2 # 3 attempts -> 2 backoff sleeps
+
+
[email protected]("on_error", ["assume_missing", "assume_exists"])
[email protected]("has_hit", [True, False])
+async def test_exists_check_mode_irrelevant_when_opensearch_answers(on_error, has_hit):
+ exists = await TaskProcessor().check_document_exists(
+ "hash", _hit_client(has_hit), on_error=on_error
+ )
+ assert exists is has_hit
+
+
+def test_verification_client_prefers_platform_writer(monkeypatch):
+ writer = MagicMock()
+ monkeypatch.setattr("config.settings.clients.opensearch", writer)
+ assert _verification_client(MagicMock()) is writer
+
+
+def test_verification_client_falls_back_when_writer_missing(monkeypatch):
+ monkeypatch.setattr("config.settings.clients.opensearch", None)
+ fallback = MagicMock()
+ assert _verification_client(fallback) is fallback