OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: post-ingestion verification must not fail files on OpenSearch 0utages (#1868)
Edwin Josec36281000243
diff --git a/src/models/processors.py b/src/models/processors.pyindex 744b46b25..dbf885d69 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -2,7 +2,7 @@ import asyncioimport mimetypesimport osimport time-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Literalfrom config.settings import clients, get_embedding_model, get_index_name, get_openrag_configfrom utils.document_processing import (@@ -29,6 +29,16 @@ if TYPE_CHECKING:from connectors.base import DocumentACL+def _verification_client(fallback_client):+ """Client for post-ingestion verification ("did the chunks land in the+ index?"). That is a system integrity check, not a user-visibility check,+ so prefer the platform writer client: it does not depend on the JWT/JWKS+ trust chain that user-scoped clients need (OpenSearch loads the backend's+ JWKS lazily, so the first user-JWT queries after a cold start can 401).+ Falls back to the caller's client when the writer is unavailable."""+ return clients.opensearch if clients.opensearch is not None else fallback_client++class TaskProcessor:"""Base class for task processors with shared processing logic"""@@ -41,10 +51,19 @@ class TaskProcessor:self,file_hash: str,opensearch_client,+ on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",) -> bool:"""Check if a document with the given hash already exists in OpenSearch.Consolidated hash checking for all processors.++ ``on_error`` picks the answer when OpenSearch stays unreachable after+ retries — the check is ambiguous then, and the safe default differs by+ caller:+ * ``"assume_missing"`` (dedupe callers): safer to reprocess than skip.+ * ``"assume_exists"`` (post-ingestion verification callers): an+ infrastructure error must not fail a file that Langflow already+ reported as ingested."""max_retries = 3retry_delay = 1.0@@ -69,7 +88,14 @@ class TaskProcessor:error=str(e),attempt=attempt + 1,)- # On final failure, assume document doesn't exist (safer to reprocess than skip)+ if on_error == "assume_exists":+ logger.warning(+ "Exists check inconclusive due to connection issues; "+ "assuming document exists",+ file_hash=file_hash,+ )+ return True+ # Safer to reprocess than skip for dedupe callers.logger.warning("Assuming document doesn't exist due to connection issues",file_hash=file_hash,@@ -85,7 +111,7 @@ class TaskProcessor:)await asyncio.sleep(retry_delay)retry_delay *= 2 # Exponential backoff- return False+ return on_error == "assume_exists"async def check_filename_exists(self,@@ -868,7 +894,11 @@ class ConnectorFileProcessor(TaskProcessor):# Langflow returns "success" even when no text was extracted# (e.g. image files without OCR). Verify the document actually# landed in OpenSearch before declaring success.- if not await self.check_document_exists(document.id, opensearch_client):+ if not await self.check_document_exists(+ document.id,+ _verification_client(opensearch_client),+ on_error="assume_exists",+ ):result = {"status": "error","error": "No text content could be extracted from document",@@ -1157,7 +1187,11 @@ class LangflowFileProcessor(TaskProcessor):# Langflow returns "success" even when no text was extracted.# Verify the document actually landed in OpenSearch.file_hash = hash_id(item)- if not await self.check_document_exists(file_hash, opensearch_client):+ if not await self.check_document_exists(+ file_hash,+ _verification_client(opensearch_client),+ on_error="assume_exists",+ ):file_task.status = TaskStatus.FAILEDfile_task.error = "No text content could be extracted from document"file_task.updated_at = time.time()diff --git a/tests/unit/test_processor_exists_verification.py b/tests/unit/test_processor_exists_verification.pynew file mode 100644index 000000000..945061ef2--- /dev/null+++ b/tests/unit/test_processor_exists_verification.py@@ -0,0 +1,100 @@+"""Post-ingestion verification must not fail files on OpenSearch outages.++#1851 added "verify the document actually landed in OpenSearch" checks after+Langflow ingestion. ``check_document_exists`` returns False on persistent+OpenSearch errors ("safer to reprocess than skip") — correct for its dedupe+callers, but the verification callers interpreted that same False as "the+chunks never landed" and marked files FAILED. A transient auth/connectivity+blip (e.g. OpenSearch's lazy JWKS load returning 401s right after startup)+then failed every in-flight file, which is what broke+test_onboarding_sample_docs in CI.++Covers the ``on_error`` mode on ``check_document_exists`` and the+``_verification_client`` selection helper.+"""++from unittest.mock import AsyncMock, MagicMock++import pytest++import models.processors as processors_module+from models.processors import TaskProcessor, _verification_client+++def _failing_client() -> MagicMock:+ client = MagicMock()+ client.search = AsyncMock(+ side_effect=ConnectionError("AuthenticationException(401, 'Unauthorized')")+ )+ return client+++def _hit_client(has_hit: bool) -> MagicMock:+ client = MagicMock()+ client.search = AsyncMock(return_value={"hits": {"hits": [{"_id": "x"}] if has_hit else []}})+ return client++[email protected](autouse=True)+def _fast_retries(monkeypatch):+ """Skip the exponential-backoff sleeps so retry exhaustion is instant.++ Returns the sleep mock so tests can assert on the retry count."""+ sleep_mock = AsyncMock()+ monkeypatch.setattr(processors_module.asyncio, "sleep", sleep_mock)+ return sleep_mock+++async def test_exists_check_error_defaults_to_missing():+ """Dedupe callers keep the historical contract: error -> reprocess."""+ exists = await TaskProcessor().check_document_exists("hash", _failing_client())+ assert exists is False+++async def test_exists_check_error_assume_exists_for_verification():+ """Verification callers must not turn an infra error into a FAILED file."""+ exists = await TaskProcessor().check_document_exists(+ "hash", _failing_client(), on_error="assume_exists"+ )+ assert exists is True+++ ("on_error", "expected"),+ [("assume_missing", False), ("assume_exists", True)],+)+async def test_exists_check_exhausts_retries_before_error_fallback(+ _fast_retries, on_error, expected+):+ """Both modes must still exhaust all retries before giving up — the+ error fallback only fires on the final attempt."""+ client = _failing_client()+ exists = await TaskProcessor().check_document_exists("hash", client, on_error=on_error)+ assert exists is expected+ assert client.search.await_count == 3 # all attempts used+ assert _fast_retries.await_count == 2 # 3 attempts -> 2 backoff sleeps++[email protected]("on_error", ["assume_missing", "assume_exists"])[email protected]("has_hit", [True, False])+async def test_exists_check_mode_irrelevant_when_opensearch_answers(on_error, has_hit):+ exists = await TaskProcessor().check_document_exists(+ "hash", _hit_client(has_hit), on_error=on_error+ )+ assert exists is has_hit+++def test_verification_client_prefers_platform_writer(monkeypatch):+ writer = MagicMock()+ monkeypatch.setattr("config.settings.clients.opensearch", writer)+ assert _verification_client(MagicMock()) is writer+++def test_verification_client_falls_back_when_writer_missing(monkeypatch):+ monkeypatch.setattr("config.settings.clients.opensearch", None)+ fallback = MagicMock()+ assert _verification_client(fallback) is fallback