Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: retry transient OpenSearch bulk failures instead of failing the … (#2356)

Nathan Bakareefb9f6ddc201
diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.py
index 71b520549..0a25a991c 100644
--- a/src/services/document_index_writer.py
+++ b/src/services/document_index_writer.py
@@ -7,6 +7,7 @@ indexing chunks into the documents index.
from __future__ import annotations
+import asyncio
import datetime
import hashlib
import json
@@ -20,6 +21,27 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+# A bulk() call executes every item independently. Thus, if one item hits a
+# transient condition (cluster under load, shard unavailable) the whole response
+# gets errors: true even though the rest of the document chunks wrote
+# successfully. This makes the entire file look like it "failed" in task
+# counters even when it's actually indexed and visible. But every op is an
+# idempotent upsert so retries are safe — only the items still failing are
+# resent, not the whole body.
+_BULK_RETRYABLE_ERROR_TYPES = frozenset(
+ {
+ "es_rejected_execution_exception",
+ "process_cluster_event_timeout_exception",
+ "unavailable_shards_exception",
+ "timeout_exception",
+ # version_conflicts are intentionally excluded — they result from
+ # concurrent writes to the same document ID, not transient failures.
+ # Retrying would just hit the same conflict again and waste the budget.
+ }
+)
+_MAX_BULK_RETRIES = 2
+_BULK_RETRY_DELAY_SECONDS = 0.5
+
@dataclass
class DocumentIndexContext:
@@ -134,7 +156,7 @@ class DocumentIndexWriter:
)
)
- result = await client.bulk(body=bulk_body, refresh=refresh)
+ result = await self._bulk_with_retry(client, bulk_body, refresh=refresh)
self._raise_for_bulk_errors(result)
if final:
await self._refresh(index_name)
@@ -315,6 +337,88 @@ class DocumentIndexWriter:
normalized["file_size"] = normalized["filesize"]
return normalized
+ @staticmethod
+ def _item_error(item: Any) -> dict[str, Any] | None:
+ action = item.get("index") or item.get("create") or item.get("update") or item
+ return action.get("error")
+
+ @classmethod
+ def _item_error_is_retryable(cls, item: Any) -> bool:
+ error = cls._item_error(item)
+ # When there is no error the item succeeded — we treat it as retryable
+ # so a successful item never blocks a retry when this runs over the
+ # still_pending list (which only ever holds items that actually failed).
+ return not error or error.get("type") in _BULK_RETRYABLE_ERROR_TYPES
+
+ async def _bulk_with_retry(
+ self, client: Any, bulk_body: list[dict[str, Any]], *, refresh: bool | str
+ ) -> Any:
+ """Only retry the items that actually failed, not the whole bulk body.
+
+ Each entry in the bulk response lines up by position with the
+ action/document pairs we sent. If we resend everything on retry we
+ end up re-submitting pairs that already succeeded — and under heavy
+ load a different subset can fail each time, meaning the file still
+ gets marked failed even though every pair succeeded at some point.
+ Instead we track each pair by its original position and only rebuild
+ the retry body from the ones still failing.
+ """
+ if len(bulk_body) % 2 != 0:
+ raise ValueError(
+ f"bulk_body must contain action/document pairs; got {len(bulk_body)} elements"
+ )
+ pairs = [tuple(bulk_body[i : i + 2]) for i in range(0, len(bulk_body), 2)]
+ pending_indices = list(range(len(pairs)))
+ final_items: list[Any] = [None] * len(pairs)
+
+ for attempt in range(_MAX_BULK_RETRIES + 1):
+ retry_body = [part for i in pending_indices for part in pairs[i]]
+ result = await client.bulk(body=retry_body, refresh=refresh)
+ if not isinstance(result, dict):
+ return result
+
+ items = result.get("items", [])
+
+ # Guard against a bad or cut-short response — the server should
+ # return exactly one item for each request we sent. If the list
+ # is short or empty we can't safely match results back to the
+ # right slots, so we treat it as an error rather than leaving
+ # gaps filled with None.
+ if len(items) != len(pending_indices):
+ raise RuntimeError(
+ f"OpenSearch bulk response returned {len(items)} items for "
+ f"{len(pending_indices)} requests — malformed response"
+ )
+
+ still_pending = []
+ for original_index, item in zip(pending_indices, items, strict=True):
+ final_items[original_index] = item
+ if self._item_error(item):
+ still_pending.append(original_index)
+
+ if not still_pending:
+ # Keep the top-level error flag as-is — if the server
+ # said errors: true but none of the individual items show
+ # an error, we still want to surface that rather than
+ # quietly calling it a success.
+ return {**result, "items": final_items, "errors": result.get("errors", False)}
+
+ all_retryable = all(
+ self._item_error_is_retryable(final_items[i]) for i in still_pending
+ )
+ if not all_retryable or attempt == _MAX_BULK_RETRIES:
+ return {**result, "items": final_items, "errors": True}
+
+ logger.warning(
+ "Retrying OpenSearch bulk write after transient error",
+ attempt=attempt + 1,
+ remaining_items=len(still_pending),
+ )
+ await asyncio.sleep(_BULK_RETRY_DELAY_SECONDS * (attempt + 1))
+ pending_indices = still_pending
+
+ return {"items": final_items, "errors": True}
+
@staticmethod
def _raise_for_bulk_errors(result: Any) -> None:
if not isinstance(result, dict) or not result.get("errors"):
diff --git a/tests/unit/test_document_index_writer_bulk_retry.py b/tests/unit/test_document_index_writer_bulk_retry.py
new file mode 100644
index 000000000..289f6cdb7
--- /dev/null
+++ b/tests/unit/test_document_index_writer_bulk_retry.py
@@ -0,0 +1,265 @@
+"""`_bulk_with_retry` must retry only the items that actually failed.
+
+A single bulk() call executes every item independently, so one item hitting a
+transient condition (e.g. the write thread pool briefly rejecting a request)
+sets `errors: true` for the whole response even though the rest of the
+document's chunks were written. Since every op is an idempotent upsert,
+retrying is safe - but resending the *whole* request would re-submit
+already-succeeded items and, under sustained load, could fail a different
+subset on each attempt, leaving the file marked failed even though every
+chunk eventually succeeded. So each retry must resend only the items still
+failing. A genuine content/schema error must still raise immediately, without
+wasting retries.
+"""
+
+from typing import Any
+
+import pytest
+
+from services.document_index_writer import DocumentIndexWriter
+
+
+class ScriptedBulkClient:
+ """Returns each scripted response in order, one per `bulk()` call, and
+ records the request bodies it was sent."""
+
+ def __init__(self, responses: list[dict[str, Any]]) -> None:
+ self.responses = responses
+ self.calls = 0
+ self.request_bodies: list[list[dict[str, Any]]] = []
+
+ async def bulk(self, *, body: list[dict[str, Any]], refresh: bool | str) -> dict[str, Any]:
+ self.request_bodies.append(body)
+ response = self.responses[min(self.calls, len(self.responses) - 1)]
+ self.calls += 1
+ return response
+
+
+def _bulk_body() -> list[dict[str, Any]]:
+ return [
+ {"index": {"_id": "chunk-1"}},
+ {"text": "a"},
+ {"index": {"_id": "chunk-2"}},
+ {"text": "b"},
+ ]
+
+
+async def test_retries_only_the_failed_item(monkeypatch):
+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)
+ client = ScriptedBulkClient(
+ [
+ {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "chunk-1", "status": 201}},
+ {
+ "index": {
+ "_id": "chunk-2",
+ "status": 429,
+ "error": {
+ "type": "es_rejected_execution_exception",
+ "reason": "queue full",
+ },
+ }
+ },
+ ],
+ },
+ {"errors": False, "items": [{"index": {"_id": "chunk-2", "status": 201}}]},
+ ]
+ )
+
+ writer = DocumentIndexWriter()
+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ assert client.calls == 2
+ assert result["errors"] is False
+ # The retry must resend only chunk-2's action/document pair, not chunk-1's.
+ assert client.request_bodies[1] == [{"index": {"_id": "chunk-2"}}, {"text": "b"}]
+ # The merged result keeps chunk-1's original successful item.
+ assert result["items"][0]["index"]["_id"] == "chunk-1"
+ assert result["items"][1]["index"]["_id"] == "chunk-2"
+
+
+async def test_does_not_retry_non_retryable_error(monkeypatch):
+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)
+ client = ScriptedBulkClient(
+ [
+ {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "chunk-1", "status": 201}},
+ {
+ "index": {
+ "_id": "chunk-2",
+ "status": 400,
+ "error": {"type": "mapper_parsing_exception", "reason": "bad field"},
+ }
+ },
+ ],
+ },
+ ]
+ )
+
+ writer = DocumentIndexWriter()
+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ assert client.calls == 1
+ assert result["errors"] is True
+
+
+async def test_gives_up_after_max_retries_still_failing(monkeypatch):
+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)
+ initial_response = {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "chunk-1", "status": 201}},
+ {
+ "index": {
+ "_id": "chunk-2",
+ "status": 429,
+ "error": {"type": "es_rejected_execution_exception", "reason": "queue full"},
+ }
+ },
+ ],
+ }
+ # Once chunk-1 has succeeded, only chunk-2 is resent on every later
+ # attempt, so those responses carry a single item.
+ persistent_chunk2_failure = {
+ "errors": True,
+ "items": [
+ {
+ "index": {
+ "_id": "chunk-2",
+ "status": 429,
+ "error": {"type": "es_rejected_execution_exception", "reason": "queue full"},
+ }
+ },
+ ],
+ }
+ client = ScriptedBulkClient([initial_response, persistent_chunk2_failure])
+
+ writer = DocumentIndexWriter()
+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ # Initial attempt + _MAX_BULK_RETRIES retries, all exhausted.
+ assert client.calls == 3
+ assert result["errors"] is True
+ # chunk-1's earlier success must survive even though the file overall failed.
+ assert result["items"][0]["index"]["_id"] == "chunk-1"
+ assert "error" not in result["items"][0]["index"]
+ # Every retry after the first call resends only chunk-2's pair.
+ assert all(
+ body == [{"index": {"_id": "chunk-2"}}, {"text": "b"}] for body in client.request_bodies[1:]
+ )
+
+
+async def test_short_items_list_raises(monkeypatch):
+ """A response with fewer items than pending requests must raise immediately
+ rather than returning None-filled slots that callers cannot safely iterate."""
+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)
+ # Two pairs were sent, but the server only returns one item.
+ client = ScriptedBulkClient(
+ [{"errors": False, "items": [{"index": {"_id": "chunk-1", "status": 201}}]}]
+ )
+
+ writer = DocumentIndexWriter()
+ with pytest.raises(RuntimeError, match="malformed response"):
+ await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ assert client.calls == 1
+
+
+async def test_top_level_errors_true_without_per_item_errors_is_preserved():
+ """If the server returns errors: true but no item carries an error dict,
+ the top-level flag must be preserved rather than overwritten with False."""
+ client = ScriptedBulkClient(
+ [
+ {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "chunk-1", "status": 201}},
+ {"index": {"_id": "chunk-2", "status": 201}},
+ ],
+ }
+ ]
+ )
+
+ writer = DocumentIndexWriter()
+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ assert client.calls == 1
+ assert result["errors"] is True
+
+
+def test_odd_length_bulk_body_raises():
+ """An odd-length bulk_body means a mismatched action/document pair — must
+ raise immediately rather than silently misaligning the request."""
+ writer = DocumentIndexWriter()
+ with pytest.raises(ValueError, match="action/document pairs"):
+ import asyncio
+
+ asyncio.get_event_loop().run_until_complete(
+ writer._bulk_with_retry(None, [{"index": {"_id": "chunk-1"}}], refresh=False)
+ )
+
+
+async def test_version_conflict_is_not_retried(monkeypatch):
+ """version_conflict_engine_exception must not be retried — conflicts are
+ caused by concurrent writes and are not transient cluster conditions."""
+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)
+ client = ScriptedBulkClient(
+ [
+ {
+ "errors": True,
+ "items": [
+ {"index": {"_id": "chunk-1", "status": 201}},
+ {
+ "index": {
+ "_id": "chunk-2",
+ "status": 409,
+ "error": {
+ "type": "version_conflict_engine_exception",
+ "reason": "document already exists",
+ },
+ }
+ },
+ ],
+ }
+ ]
+ )
+
+ writer = DocumentIndexWriter()
+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)
+
+ assert client.calls == 1
+ assert result["errors"] is True
+
+
+async def test_item_error_fallback_for_unknown_action_type():
+ """_item_error must still extract the error dict when the item does not
+ use a recognised action key (index/create/update), falling back to the
+ item itself."""
+ writer = DocumentIndexWriter()
+ item_with_unknown_action = {
+ "delete": {
+ "_id": "chunk-1",
+ "status": 404,
+ "error": {"type": "not_found", "reason": "document missing"},
+ }
+ }
+ # The fallback `or item` path: none of index/create/update match, so
+ # _item_error receives the raw item dict and looks for "error" on it
+ # directly — which is absent at the top level, so returns None.
+ assert writer._item_error(item_with_unknown_action) is None
+
+ # Confirm an item that truly has no recognised action and carries a
+ # top-level error key is picked up by the fallback.
+ bare_error_item: dict[str, Any] = {"error": {"type": "some_exception", "reason": "boom"}}
+ assert writer._item_error(bare_error_item) == {"type": "some_exception", "reason": "boom"}
diff --git a/tests/unit/test_document_index_writer_ownership.py b/tests/unit/test_document_index_writer_ownership.py
index deab0e326..5682998dc 100644
--- a/tests/unit/test_document_index_writer_ownership.py
+++ b/tests/unit/test_document_index_writer_ownership.py
@@ -38,10 +38,12 @@ class InMemoryOpenSearch:
self.indices = InMemoryIndices()
async def bulk(self, *, body: list[dict[str, Any]], refresh: bool | str) -> dict[str, Any]:
+ items = []
for offset in range(0, len(body), 2):
document_id = body[offset]["index"]["_id"]
self.documents[document_id] = body[offset + 1]
- return {"errors": False}
+ items.append({"index": {"_id": document_id, "status": 200}})
+ return {"errors": False, "items": items}
def visible_documents(self, owner: str) -> list[dict[str, Any]]:
return [document for document in self.documents.values() if document.get("owner") == owner]