Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Address 409 error for bucket connectors (#2349)

Mike Fortman6719338ecf28
diff --git a/frontend/lib/chat-stream-errors.test.ts b/frontend/lib/chat-stream-errors.test.ts
index a0c508a4b..8b3caa832 100644
--- a/frontend/lib/chat-stream-errors.test.ts
+++ b/frontend/lib/chat-stream-errors.test.ts
@@ -172,4 +172,20 @@ describe("formatProviderErrorMessage", () => {
);
assert.equal(formatProviderErrorMessage("Error: boom"), "Error: boom");
});
+
+ it("keeps the whole string when the prefix is only a syntactic fragment", () => {
+ const body =
+ '{"took":2719,"timed_out":false,"total":300,"updated":19,' +
+ '"version_conflicts":1,"failures":[{"index":"documents","id":"20",' +
+ '"cause":{"type":"version_conflict_engine_exception"}}]}';
+ const raw = `ConflictError(409, '${body}')`;
+ assert.equal(formatProviderErrorMessage(raw), raw);
+ });
+
+ it("still keeps a readable prefix that is a real sentence", () => {
+ assert.equal(
+ formatProviderErrorMessage("Invalid API key {not-valid-json"),
+ "Invalid API key",
+ );
+ });
});
diff --git a/frontend/lib/chat-stream-errors.ts b/frontend/lib/chat-stream-errors.ts
index 6ea5edacf..09d73618f 100644
--- a/frontend/lib/chat-stream-errors.ts
+++ b/frontend/lib/chat-stream-errors.ts
@@ -174,6 +174,19 @@ function humanizeProviderErrorMessage(message: string): string {
return message;
}
+/**
+ * A prefix ending on an unclosed delimiter is a syntactic fragment, not a message.
+ * `ConflictError(409, '` is the whole of what precedes the JSON in an opensearch-py
+ * transport error, and the payload after that quote is the entire diagnosis — so
+ * keeping only the prefix discards everything useful. A real prefix ("Invalid API
+ * key {...") ends on a word character and is still preferred.
+ */
+const SYNTACTIC_FRAGMENT_TAIL = /[([{'"`,=:]$/;
+
+function isReadablePrefix(prefix: string): boolean {
+ return prefix.length > 0 && !SYNTACTIC_FRAGMENT_TAIL.test(prefix);
+}
+
/**
* Strip embedded provider JSON payloads so chat never shows raw error objects.
*/
@@ -202,7 +215,7 @@ export function formatProviderErrorMessage(text: string): string {
.replace(/[:\s]+$/, "")
.trim(),
);
- if (prefix) {
+ if (isReadablePrefix(prefix)) {
return humanizeProviderErrorMessage(prefix);
}
}
@@ -216,7 +229,7 @@ export function formatProviderErrorMessage(text: string): string {
.replace(/[:\s]+$/, "")
.trim(),
);
- if (prefix) {
+ if (isReadablePrefix(prefix)) {
return humanizeProviderErrorMessage(prefix);
}
}
diff --git a/frontend/lib/task-error-display.test.ts b/frontend/lib/task-error-display.test.ts
new file mode 100644
index 000000000..03d9825dd
--- /dev/null
+++ b/frontend/lib/task-error-display.test.ts
@@ -0,0 +1,80 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import type { TaskFileEntry } from "@/app/api/queries/useGetTasksQuery";
+import { resolveTaskFileError } from "@/lib/task-error-display";
+
+// A real _update_by_query 409 as opensearch-py renders it: the whole response
+// body becomes the exception message, because that response carries no
+// top-level "error" key for the client to use instead.
+const OPENSEARCH_CONFLICT_ERROR =
+ 'ConflictError(409, \'{"took":2719,"timed_out":false,"total":300,' +
+ '"updated":19,"version_conflicts":1,"failures":[{"index":"documents",' +
+ '"id":"20","cause":{"type":"version_conflict_engine_exception"}}]}\')';
+
+const BACKEND_CLASSIFIED_MESSAGE =
+ "The search index was busy and this change conflicted with another " +
+ "update. Nothing is wrong with the file — retry ingestion.";
+
+describe("resolveTaskFileError", () => {
+ it("prefers the backend's user_facing_message over the raw error", () => {
+ // This precedence is what makes the backend classifier work at all. If these
+ // checks are ever reordered, a classified OpenSearch failure would fall back
+ // to the raw payload and be mangled again.
+ const entry: TaskFileEntry = {
+ status: "failed",
+ error: OPENSEARCH_CONFLICT_ERROR,
+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,
+ };
+
+ assert.equal(resolveTaskFileError(entry), BACKEND_CLASSIFIED_MESSAGE);
+ });
+
+ it("passes a clean sentence through the provider parser unchanged", () => {
+ // The message contains no JSON, so the chat provider-error parser must not
+ // rewrite or truncate it.
+ const entry: TaskFileEntry = {
+ status: "failed",
+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,
+ };
+
+ assert.equal(resolveTaskFileError(entry), BACKEND_CLASSIFIED_MESSAGE);
+ });
+
+ it("keeps the whole raw error when the backend did not classify it", () => {
+ // Regression guard: this used to collapse to the 20-character fragment
+ // "ConflictError(409, '", discarding the failures[] array that names the
+ // index, shard and document — the only way to diagnose the failure.
+ const entry: TaskFileEntry = {
+ status: "failed",
+ error: OPENSEARCH_CONFLICT_ERROR,
+ };
+
+ const resolved = resolveTaskFileError(entry);
+ assert.equal(resolved, OPENSEARCH_CONFLICT_ERROR);
+ assert.ok(resolved.includes("version_conflict_engine_exception"));
+ });
+
+ it("still prefers a result warning over everything else", () => {
+ const entry: TaskFileEntry = {
+ status: "skipped",
+ error: OPENSEARCH_CONFLICT_ERROR,
+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,
+ result: {
+ warning: "File no longer exists at source; removed from index.",
+ },
+ };
+
+ assert.equal(
+ resolveTaskFileError(entry),
+ "File no longer exists at source; removed from index.",
+ );
+ });
+
+ it("falls back to the task-level error, then to Unknown error", () => {
+ assert.equal(
+ resolveTaskFileError({ status: "failed" }, "task blew up"),
+ "task blew up",
+ );
+ assert.equal(resolveTaskFileError({ status: "failed" }), "Unknown error");
+ });
+});
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 3d6398a09..7d1153599 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -1,6 +1,7 @@
import asyncio
import os
import random
+import re
import time
import traceback
import uuid
@@ -91,6 +92,60 @@ def _is_task_cancellation_error(error: str) -> bool:
return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)
+# opensearch-py renders transport failures as "ClassName(<status>, ...)". For the
+# by-query APIs (_update_by_query / _delete_by_query) the response carries no
+# top-level "error" key, so Connection._raise_error falls back to using the whole
+# JSON response body as the exception message — several hundred characters of
+# payload where a reason string would normally be.
+#
+# Classifying these early matters twice over: an infrastructure fault never
+# reaches the UI as a raw exception repr, and the JSON body cannot trip the
+# substring heuristics further down (a mapper_parsing_exception body contains
+# "failed to parse", which would otherwise read as a corrupted file).
+#
+# ConnectionError / ConnectionTimeout are deliberately excluded: they render with
+# a different __str__ and are already covered by _is_transient_connectivity_error.
+_OPENSEARCH_TRANSPORT_ERROR_RE = re.compile(
+ r"\b(?:ConflictError|NotFoundError|RequestError|TransportError"
+ r"|AuthenticationException|AuthorizationException)\((\d{3})[,)]"
+)
+
+
+def _opensearch_failure_metadata(error: str) -> dict | None:
+ """Classify an opensearch-py transport failure surfaced as a task error.
+
+ Returns None when the error is not an OpenSearch transport failure, so the
+ caller falls through to the existing classification branches.
+ """
+ if not error:
+ return None
+ match = _OPENSEARCH_TRANSPORT_ERROR_RE.search(error)
+ if not match:
+ return None
+
+ status = match.group(1)
+ if status == "409":
+ message = (
+ "The search index was busy and this change conflicted with another "
+ "update. Nothing is wrong with the file — retry ingestion."
+ )
+ else:
+ message = (
+ f"The search index rejected this document (OpenSearch error {status}). "
+ "This is an infrastructure problem, not a problem with the file. "
+ "Retry ingestion, and check the backend logs if it persists."
+ )
+ # RETRYABLE across the board: of the two available values, USER_ACTIONABLE
+ # would be actively wrong — there is nothing the uploader can change about
+ # their file to fix a search-index fault.
+ return {
+ "component": "opensearch",
+ "failure_phase": "indexing",
+ "user_facing_message": message,
+ "actionable_by": "RETRYABLE",
+ }
+
+
def _provider_credential_failure_metadata(error: str) -> dict | None:
"""Classify invalid/revoked provider API key failures for ingestion toasts/UI."""
if not error:
@@ -1020,6 +1075,13 @@ class TaskService:
"actionable_by": "USER_ACTIONABLE",
}
+ # Before any substring heuristic: an OpenSearch transport failure carries a
+ # JSON payload whose contents would otherwise match the file-corruption and
+ # "already exists" markers below.
+ opensearch_meta = _opensearch_failure_metadata(error)
+ if opensearch_meta:
+ return opensearch_meta
+
if "incorrect password" in error.lower(): # for password protected pdf cases
return {
"component": "docling",
diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.py
index 697862547..8dd228896 100644
--- a/tests/unit/test_task_service_get_task_status2.py
+++ b/tests/unit/test_task_service_get_task_status2.py
@@ -14,7 +14,7 @@ from unittest.mock import Mock
import pytest
from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask
-from services.task_service import TaskService
+from services.task_service import TaskService, _opensearch_failure_metadata
# ---------------------------------------------------------------------------
# Fixtures
@@ -671,3 +671,151 @@ class TestGetTaskStatusRegression:
for key in v1_file:
assert key in v2_file, f"get_task_status2 file entry missing key: {key}"
assert v1_file[key] == v2_file[key], f"file entry key '{key}' differs"
+
+
+# ---------------------------------------------------------------------------
+# OpenSearch transport failures
+# ---------------------------------------------------------------------------
+
+
+class TestOpenSearchFailureMetadata:
+ def test_by_query_conflict_is_classified_as_retryable_opensearch_failure(self, task_service):
+ # opensearch-py puts the entire by-query response body into the message,
+ # because that response has no top-level "error" key to use instead.
+ ft = _make_file_task(
+ filename="logo.png",
+ phase=IngestionPhase.LANGFLOW,
+ error=(
+ 'ConflictError(409, \'{"took":2719,"timed_out":false,"total":300,'
+ '"updated":19,"version_conflicts":1,"failures":[{"index":"documents",'
+ '"id":"20","cause":{"type":"version_conflict_engine_exception"}}]}\')'
+ ),
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "opensearch"
+ assert meta["failure_phase"] == "indexing"
+ assert meta["actionable_by"] == "RETRYABLE"
+ msg = meta["user_facing_message"]
+ assert "retry ingestion" in msg.lower()
+ # The raw payload must not reach the UI.
+ assert "version_conflict_engine_exception" not in msg
+ assert "{" not in msg
+
+ def test_mapper_parsing_body_is_not_read_as_a_corrupt_file(self, task_service):
+ # A 400 body contains the substring "failed to parse", which
+ # _is_non_retryable_file_error would otherwise classify as a corrupt file.
+ ft = _make_file_task(
+ error="RequestError(400, 'mapper_parsing_exception', 'failed to parse field [x]')"
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "opensearch"
+ assert "corrupt" not in meta["user_facing_message"].lower()
+
+ def test_image_without_text_still_gets_the_ocr_guidance(self, task_service):
+ ft = _make_file_task(
+ filename="scan.png",
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error="No text content could be extracted from document",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+ assert "ocr" in meta["user_facing_message"].lower()
+
+ def test_plain_errors_are_left_to_the_existing_branches(self, task_service):
+ ft = _make_file_task(error="Docling conversion did not complete (timeout): 300s")
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+
+
+class TestOpenSearchErrorStringContract:
+ """Pin the opensearch-py rendering the classifier depends on.
+
+ _opensearch_failure_metadata matches the exception's *string form*, which is
+ built by opensearch-py's Connection._raise_error, not by us. If a client
+ upgrade changes that rendering, the classifier silently stops matching and
+ users start seeing raw payloads again. These tests fail loudly instead of
+ letting that regress quietly.
+ """
+
+ def test_by_query_conflict_renders_body_as_the_message(self):
+ from opensearchpy.connection.base import Connection
+ from opensearchpy.exceptions import ConflictError
+
+ # An _update_by_query 409 carries no top-level "error" key, so
+ # _raise_error falls back to using the whole response body as the message.
+ body = (
+ '{"took":12,"timed_out":false,"total":3,"updated":1,'
+ '"version_conflicts":1,"failures":[{"index":"documents","id":"7",'
+ '"cause":{"type":"version_conflict_engine_exception"}}]}'
+ )
+ with pytest.raises(ConflictError) as excinfo:
+ Connection()._raise_error(409, body, "application/json")
+
+ rendered = str(excinfo.value)
+ assert rendered.startswith("ConflictError(409, '{")
+ meta = _opensearch_failure_metadata(rendered)
+ assert meta is not None
+ assert meta["component"] == "opensearch"
+ assert meta["actionable_by"] == "RETRYABLE"
+
+ def test_single_document_conflict_renders_a_short_reason(self):
+ from opensearchpy.connection.base import Connection
+ from opensearchpy.exceptions import ConflictError
+
+ # Single-document writes render differently — a reason string rather than
+ # a JSON body — and must classify identically.
+ body = (
+ '{"error":{"root_cause":[{"type":"version_conflict_engine_exception",'
+ '"reason":"[7]: version conflict"}],'
+ '"type":"version_conflict_engine_exception"},"status":409}'
+ )
+ with pytest.raises(ConflictError) as excinfo:
+ Connection()._raise_error(409, body, "application/json")
+
+ rendered = str(excinfo.value)
+ assert rendered.startswith("ConflictError(409, 'version_conflict_engine_exception'")
+ meta = _opensearch_failure_metadata(rendered)
+ assert meta is not None
+ assert meta["component"] == "opensearch"
+
+ def test_non_opensearch_text_is_not_classified(self):
+ for text in (
+ "",
+ "Docling conversion did not complete (timeout): polling exceeded 300s",
+ "the sync hit a ConflictError while running", # prose, no (status)
+ ):
+ assert _opensearch_failure_metadata(text) is None
+
+
+class TestGetTaskStatus2OpenSearchFailure:
+ def test_conflict_surfaces_as_structured_metadata(self, task_service):
+ ft = _make_file_task(
+ filename="logo.png",
+ phase=IngestionPhase.LANGFLOW,
+ error=(
+ 'ConflictError(409, \'{"took":9,"total":3,"updated":1,'
+ '"version_conflicts":1,"failures":[{"index":"documents","id":"7"}]}\')'
+ ),
+ )
+ task = _make_upload_task("os409", {"logo.png": ft})
+ task.status = TaskStatus.FAILED
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ file_entry = task_service.get_task_status2("user1", "os409")["files"]["logo.png"]
+
+ assert file_entry["status"] == "failed"
+ assert file_entry["component"] == "opensearch"
+ assert file_entry["failure_phase"] == "indexing"
+ assert file_entry["actionable_by"] == "RETRYABLE"
+ # The payload must not reach the user as the displayed message...
+ assert "{" not in file_entry["user_facing_message"]
+ assert "version_conflicts" not in file_entry["user_facing_message"]
+ # ...but the raw error stays on the entry, which is the only way to
+ # diagnose which OpenSearch operation actually failed.
+ assert file_entry["error"].startswith("ConflictError(409, '{")