OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: retry transient OpenSearch bulk failures instead of failing the … (#2356)
Nathan Bakareefb9f6ddc201
diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.pyindex 71b520549..0a25a991c 100644--- a/src/services/document_index_writer.py+++ b/src/services/document_index_writer.py@@ -7,6 +7,7 @@ indexing chunks into the documents index.from __future__ import annotations+import asyncioimport datetimeimport hashlibimport json@@ -20,6 +21,27 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)+# A bulk() call executes every item independently. Thus, if one item hits a+# transient condition (cluster under load, shard unavailable) the whole response+# gets errors: true even though the rest of the document chunks wrote+# successfully. This makes the entire file look like it "failed" in task+# counters even when it's actually indexed and visible. But every op is an+# idempotent upsert so retries are safe — only the items still failing are+# resent, not the whole body.+_BULK_RETRYABLE_ERROR_TYPES = frozenset(+ {+ "es_rejected_execution_exception",+ "process_cluster_event_timeout_exception",+ "unavailable_shards_exception",+ "timeout_exception",+ # version_conflicts are intentionally excluded — they result from+ # concurrent writes to the same document ID, not transient failures.+ # Retrying would just hit the same conflict again and waste the budget.+ }+)+_MAX_BULK_RETRIES = 2+_BULK_RETRY_DELAY_SECONDS = 0.5+@dataclassclass DocumentIndexContext:@@ -134,7 +156,7 @@ class DocumentIndexWriter:))- result = await client.bulk(body=bulk_body, refresh=refresh)+ result = await self._bulk_with_retry(client, bulk_body, refresh=refresh)self._raise_for_bulk_errors(result)if final:await self._refresh(index_name)@@ -315,6 +337,88 @@ class DocumentIndexWriter:normalized["file_size"] = normalized["filesize"]return normalized+ @staticmethod+ def _item_error(item: Any) -> dict[str, Any] | None:+ action = item.get("index") or item.get("create") or item.get("update") or item+ return action.get("error")++ @classmethod+ def _item_error_is_retryable(cls, item: Any) -> bool:+ error = cls._item_error(item)+ # When there is no error the item succeeded — we treat it as retryable+ # so a successful item never blocks a retry when this runs over the+ # still_pending list (which only ever holds items that actually failed).+ return not error or error.get("type") in _BULK_RETRYABLE_ERROR_TYPES++ async def _bulk_with_retry(+ self, client: Any, bulk_body: list[dict[str, Any]], *, refresh: bool | str+ ) -> Any:+ """Only retry the items that actually failed, not the whole bulk body.++ Each entry in the bulk response lines up by position with the+ action/document pairs we sent. If we resend everything on retry we+ end up re-submitting pairs that already succeeded — and under heavy+ load a different subset can fail each time, meaning the file still+ gets marked failed even though every pair succeeded at some point.+ Instead we track each pair by its original position and only rebuild+ the retry body from the ones still failing.+ """+ if len(bulk_body) % 2 != 0:+ raise ValueError(+ f"bulk_body must contain action/document pairs; got {len(bulk_body)} elements"+ )+ pairs = [tuple(bulk_body[i : i + 2]) for i in range(0, len(bulk_body), 2)]+ pending_indices = list(range(len(pairs)))+ final_items: list[Any] = [None] * len(pairs)++ for attempt in range(_MAX_BULK_RETRIES + 1):+ retry_body = [part for i in pending_indices for part in pairs[i]]+ result = await client.bulk(body=retry_body, refresh=refresh)+ if not isinstance(result, dict):+ return result++ items = result.get("items", [])++ # Guard against a bad or cut-short response — the server should+ # return exactly one item for each request we sent. If the list+ # is short or empty we can't safely match results back to the+ # right slots, so we treat it as an error rather than leaving+ # gaps filled with None.+ if len(items) != len(pending_indices):+ raise RuntimeError(+ f"OpenSearch bulk response returned {len(items)} items for "+ f"{len(pending_indices)} requests — malformed response"+ )++ still_pending = []+ for original_index, item in zip(pending_indices, items, strict=True):+ final_items[original_index] = item+ if self._item_error(item):+ still_pending.append(original_index)++ if not still_pending:+ # Keep the top-level error flag as-is — if the server+ # said errors: true but none of the individual items show+ # an error, we still want to surface that rather than+ # quietly calling it a success.+ return {**result, "items": final_items, "errors": result.get("errors", False)}++ all_retryable = all(+ self._item_error_is_retryable(final_items[i]) for i in still_pending+ )+ if not all_retryable or attempt == _MAX_BULK_RETRIES:+ return {**result, "items": final_items, "errors": True}++ logger.warning(+ "Retrying OpenSearch bulk write after transient error",+ attempt=attempt + 1,+ remaining_items=len(still_pending),+ )+ await asyncio.sleep(_BULK_RETRY_DELAY_SECONDS * (attempt + 1))+ pending_indices = still_pending++ return {"items": final_items, "errors": True}+@staticmethoddef _raise_for_bulk_errors(result: Any) -> None:if not isinstance(result, dict) or not result.get("errors"):diff --git a/tests/unit/test_document_index_writer_bulk_retry.py b/tests/unit/test_document_index_writer_bulk_retry.pynew file mode 100644index 000000000..289f6cdb7--- /dev/null+++ b/tests/unit/test_document_index_writer_bulk_retry.py@@ -0,0 +1,265 @@+"""`_bulk_with_retry` must retry only the items that actually failed.++A single bulk() call executes every item independently, so one item hitting a+transient condition (e.g. the write thread pool briefly rejecting a request)+sets `errors: true` for the whole response even though the rest of the+document's chunks were written. Since every op is an idempotent upsert,+retrying is safe - but resending the *whole* request would re-submit+already-succeeded items and, under sustained load, could fail a different+subset on each attempt, leaving the file marked failed even though every+chunk eventually succeeded. So each retry must resend only the items still+failing. A genuine content/schema error must still raise immediately, without+wasting retries.+"""++from typing import Any++import pytest++from services.document_index_writer import DocumentIndexWriter+++class ScriptedBulkClient:+ """Returns each scripted response in order, one per `bulk()` call, and+ records the request bodies it was sent."""++ def __init__(self, responses: list[dict[str, Any]]) -> None:+ self.responses = responses+ self.calls = 0+ self.request_bodies: list[list[dict[str, Any]]] = []++ async def bulk(self, *, body: list[dict[str, Any]], refresh: bool | str) -> dict[str, Any]:+ self.request_bodies.append(body)+ response = self.responses[min(self.calls, len(self.responses) - 1)]+ self.calls += 1+ return response+++def _bulk_body() -> list[dict[str, Any]]:+ return [+ {"index": {"_id": "chunk-1"}},+ {"text": "a"},+ {"index": {"_id": "chunk-2"}},+ {"text": "b"},+ ]+++async def test_retries_only_the_failed_item(monkeypatch):+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)+ client = ScriptedBulkClient(+ [+ {+ "errors": True,+ "items": [+ {"index": {"_id": "chunk-1", "status": 201}},+ {+ "index": {+ "_id": "chunk-2",+ "status": 429,+ "error": {+ "type": "es_rejected_execution_exception",+ "reason": "queue full",+ },+ }+ },+ ],+ },+ {"errors": False, "items": [{"index": {"_id": "chunk-2", "status": 201}}]},+ ]+ )++ writer = DocumentIndexWriter()+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ assert client.calls == 2+ assert result["errors"] is False+ # The retry must resend only chunk-2's action/document pair, not chunk-1's.+ assert client.request_bodies[1] == [{"index": {"_id": "chunk-2"}}, {"text": "b"}]+ # The merged result keeps chunk-1's original successful item.+ assert result["items"][0]["index"]["_id"] == "chunk-1"+ assert result["items"][1]["index"]["_id"] == "chunk-2"+++async def test_does_not_retry_non_retryable_error(monkeypatch):+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)+ client = ScriptedBulkClient(+ [+ {+ "errors": True,+ "items": [+ {"index": {"_id": "chunk-1", "status": 201}},+ {+ "index": {+ "_id": "chunk-2",+ "status": 400,+ "error": {"type": "mapper_parsing_exception", "reason": "bad field"},+ }+ },+ ],+ },+ ]+ )++ writer = DocumentIndexWriter()+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ assert client.calls == 1+ assert result["errors"] is True+++async def test_gives_up_after_max_retries_still_failing(monkeypatch):+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)+ initial_response = {+ "errors": True,+ "items": [+ {"index": {"_id": "chunk-1", "status": 201}},+ {+ "index": {+ "_id": "chunk-2",+ "status": 429,+ "error": {"type": "es_rejected_execution_exception", "reason": "queue full"},+ }+ },+ ],+ }+ # Once chunk-1 has succeeded, only chunk-2 is resent on every later+ # attempt, so those responses carry a single item.+ persistent_chunk2_failure = {+ "errors": True,+ "items": [+ {+ "index": {+ "_id": "chunk-2",+ "status": 429,+ "error": {"type": "es_rejected_execution_exception", "reason": "queue full"},+ }+ },+ ],+ }+ client = ScriptedBulkClient([initial_response, persistent_chunk2_failure])++ writer = DocumentIndexWriter()+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ # Initial attempt + _MAX_BULK_RETRIES retries, all exhausted.+ assert client.calls == 3+ assert result["errors"] is True+ # chunk-1's earlier success must survive even though the file overall failed.+ assert result["items"][0]["index"]["_id"] == "chunk-1"+ assert "error" not in result["items"][0]["index"]+ # Every retry after the first call resends only chunk-2's pair.+ assert all(+ body == [{"index": {"_id": "chunk-2"}}, {"text": "b"}] for body in client.request_bodies[1:]+ )+++async def test_short_items_list_raises(monkeypatch):+ """A response with fewer items than pending requests must raise immediately+ rather than returning None-filled slots that callers cannot safely iterate."""+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)+ # Two pairs were sent, but the server only returns one item.+ client = ScriptedBulkClient(+ [{"errors": False, "items": [{"index": {"_id": "chunk-1", "status": 201}}]}]+ )++ writer = DocumentIndexWriter()+ with pytest.raises(RuntimeError, match="malformed response"):+ await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ assert client.calls == 1+++async def test_top_level_errors_true_without_per_item_errors_is_preserved():+ """If the server returns errors: true but no item carries an error dict,+ the top-level flag must be preserved rather than overwritten with False."""+ client = ScriptedBulkClient(+ [+ {+ "errors": True,+ "items": [+ {"index": {"_id": "chunk-1", "status": 201}},+ {"index": {"_id": "chunk-2", "status": 201}},+ ],+ }+ ]+ )++ writer = DocumentIndexWriter()+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ assert client.calls == 1+ assert result["errors"] is True+++def test_odd_length_bulk_body_raises():+ """An odd-length bulk_body means a mismatched action/document pair — must+ raise immediately rather than silently misaligning the request."""+ writer = DocumentIndexWriter()+ with pytest.raises(ValueError, match="action/document pairs"):+ import asyncio++ asyncio.get_event_loop().run_until_complete(+ writer._bulk_with_retry(None, [{"index": {"_id": "chunk-1"}}], refresh=False)+ )+++async def test_version_conflict_is_not_retried(monkeypatch):+ """version_conflict_engine_exception must not be retried — conflicts are+ caused by concurrent writes and are not transient cluster conditions."""+ monkeypatch.setattr("services.document_index_writer._BULK_RETRY_DELAY_SECONDS", 0)+ client = ScriptedBulkClient(+ [+ {+ "errors": True,+ "items": [+ {"index": {"_id": "chunk-1", "status": 201}},+ {+ "index": {+ "_id": "chunk-2",+ "status": 409,+ "error": {+ "type": "version_conflict_engine_exception",+ "reason": "document already exists",+ },+ }+ },+ ],+ }+ ]+ )++ writer = DocumentIndexWriter()+ result = await writer._bulk_with_retry(client, _bulk_body(), refresh=False)++ assert client.calls == 1+ assert result["errors"] is True+++async def test_item_error_fallback_for_unknown_action_type():+ """_item_error must still extract the error dict when the item does not+ use a recognised action key (index/create/update), falling back to the+ item itself."""+ writer = DocumentIndexWriter()+ item_with_unknown_action = {+ "delete": {+ "_id": "chunk-1",+ "status": 404,+ "error": {"type": "not_found", "reason": "document missing"},+ }+ }+ # The fallback `or item` path: none of index/create/update match, so+ # _item_error receives the raw item dict and looks for "error" on it+ # directly — which is absent at the top level, so returns None.+ assert writer._item_error(item_with_unknown_action) is None++ # Confirm an item that truly has no recognised action and carries a+ # top-level error key is picked up by the fallback.+ bare_error_item: dict[str, Any] = {"error": {"type": "some_exception", "reason": "boom"}}+ assert writer._item_error(bare_error_item) == {"type": "some_exception", "reason": "boom"}diff --git a/tests/unit/test_document_index_writer_ownership.py b/tests/unit/test_document_index_writer_ownership.pyindex deab0e326..5682998dc 100644--- a/tests/unit/test_document_index_writer_ownership.py+++ b/tests/unit/test_document_index_writer_ownership.py@@ -38,10 +38,12 @@ class InMemoryOpenSearch:self.indices = InMemoryIndices()async def bulk(self, *, body: list[dict[str, Any]], refresh: bool | str) -> dict[str, Any]:+ items = []for offset in range(0, len(body), 2):document_id = body[offset]["index"]["_id"]self.documents[document_id] = body[offset + 1]- return {"errors": False}+ items.append({"index": {"_id": document_id, "status": 200}})+ return {"errors": False, "items": items}def visible_documents(self, owner: str) -> list[dict[str, Any]]:return [document for document in self.documents.values() if document.get("owner") == owner]