OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Address 409 error for bucket connectors (#2349)
Mike Fortman6719338ecf28
diff --git a/frontend/lib/chat-stream-errors.test.ts b/frontend/lib/chat-stream-errors.test.tsindex a0c508a4b..8b3caa832 100644--- a/frontend/lib/chat-stream-errors.test.ts+++ b/frontend/lib/chat-stream-errors.test.ts@@ -172,4 +172,20 @@ describe("formatProviderErrorMessage", () => {);assert.equal(formatProviderErrorMessage("Error: boom"), "Error: boom");});++ it("keeps the whole string when the prefix is only a syntactic fragment", () => {+ const body =+ '{"took":2719,"timed_out":false,"total":300,"updated":19,' ++ '"version_conflicts":1,"failures":[{"index":"documents","id":"20",' ++ '"cause":{"type":"version_conflict_engine_exception"}}]}';+ const raw = `ConflictError(409, '${body}')`;+ assert.equal(formatProviderErrorMessage(raw), raw);+ });++ it("still keeps a readable prefix that is a real sentence", () => {+ assert.equal(+ formatProviderErrorMessage("Invalid API key {not-valid-json"),+ "Invalid API key",+ );+ });});diff --git a/frontend/lib/chat-stream-errors.ts b/frontend/lib/chat-stream-errors.tsindex 6ea5edacf..09d73618f 100644--- a/frontend/lib/chat-stream-errors.ts+++ b/frontend/lib/chat-stream-errors.ts@@ -174,6 +174,19 @@ function humanizeProviderErrorMessage(message: string): string {return message;}+/**+ * A prefix ending on an unclosed delimiter is a syntactic fragment, not a message.+ * `ConflictError(409, '` is the whole of what precedes the JSON in an opensearch-py+ * transport error, and the payload after that quote is the entire diagnosis — so+ * keeping only the prefix discards everything useful. A real prefix ("Invalid API+ * key {...") ends on a word character and is still preferred.+ */+const SYNTACTIC_FRAGMENT_TAIL = /[([{'"`,=:]$/;++function isReadablePrefix(prefix: string): boolean {+ return prefix.length > 0 && !SYNTACTIC_FRAGMENT_TAIL.test(prefix);+}+/*** Strip embedded provider JSON payloads so chat never shows raw error objects.*/@@ -202,7 +215,7 @@ export function formatProviderErrorMessage(text: string): string {.replace(/[:\s]+$/, "").trim(),);- if (prefix) {+ if (isReadablePrefix(prefix)) {return humanizeProviderErrorMessage(prefix);}}@@ -216,7 +229,7 @@ export function formatProviderErrorMessage(text: string): string {.replace(/[:\s]+$/, "").trim(),);- if (prefix) {+ if (isReadablePrefix(prefix)) {return humanizeProviderErrorMessage(prefix);}}diff --git a/frontend/lib/task-error-display.test.ts b/frontend/lib/task-error-display.test.tsnew file mode 100644index 000000000..03d9825dd--- /dev/null+++ b/frontend/lib/task-error-display.test.ts@@ -0,0 +1,80 @@+import assert from "node:assert/strict";+import { describe, it } from "node:test";+import type { TaskFileEntry } from "@/app/api/queries/useGetTasksQuery";+import { resolveTaskFileError } from "@/lib/task-error-display";++// A real _update_by_query 409 as opensearch-py renders it: the whole response+// body becomes the exception message, because that response carries no+// top-level "error" key for the client to use instead.+const OPENSEARCH_CONFLICT_ERROR =+ 'ConflictError(409, \'{"took":2719,"timed_out":false,"total":300,' ++ '"updated":19,"version_conflicts":1,"failures":[{"index":"documents",' ++ '"id":"20","cause":{"type":"version_conflict_engine_exception"}}]}\')';++const BACKEND_CLASSIFIED_MESSAGE =+ "The search index was busy and this change conflicted with another " ++ "update. Nothing is wrong with the file — retry ingestion.";++describe("resolveTaskFileError", () => {+ it("prefers the backend's user_facing_message over the raw error", () => {+ // This precedence is what makes the backend classifier work at all. If these+ // checks are ever reordered, a classified OpenSearch failure would fall back+ // to the raw payload and be mangled again.+ const entry: TaskFileEntry = {+ status: "failed",+ error: OPENSEARCH_CONFLICT_ERROR,+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,+ };++ assert.equal(resolveTaskFileError(entry), BACKEND_CLASSIFIED_MESSAGE);+ });++ it("passes a clean sentence through the provider parser unchanged", () => {+ // The message contains no JSON, so the chat provider-error parser must not+ // rewrite or truncate it.+ const entry: TaskFileEntry = {+ status: "failed",+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,+ };++ assert.equal(resolveTaskFileError(entry), BACKEND_CLASSIFIED_MESSAGE);+ });++ it("keeps the whole raw error when the backend did not classify it", () => {+ // Regression guard: this used to collapse to the 20-character fragment+ // "ConflictError(409, '", discarding the failures[] array that names the+ // index, shard and document — the only way to diagnose the failure.+ const entry: TaskFileEntry = {+ status: "failed",+ error: OPENSEARCH_CONFLICT_ERROR,+ };++ const resolved = resolveTaskFileError(entry);+ assert.equal(resolved, OPENSEARCH_CONFLICT_ERROR);+ assert.ok(resolved.includes("version_conflict_engine_exception"));+ });++ it("still prefers a result warning over everything else", () => {+ const entry: TaskFileEntry = {+ status: "skipped",+ error: OPENSEARCH_CONFLICT_ERROR,+ user_facing_message: BACKEND_CLASSIFIED_MESSAGE,+ result: {+ warning: "File no longer exists at source; removed from index.",+ },+ };++ assert.equal(+ resolveTaskFileError(entry),+ "File no longer exists at source; removed from index.",+ );+ });++ it("falls back to the task-level error, then to Unknown error", () => {+ assert.equal(+ resolveTaskFileError({ status: "failed" }, "task blew up"),+ "task blew up",+ );+ assert.equal(resolveTaskFileError({ status: "failed" }), "Unknown error");+ });+});diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 3d6398a09..7d1153599 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -1,6 +1,7 @@import asyncioimport osimport random+import reimport timeimport tracebackimport uuid@@ -91,6 +92,60 @@ def _is_task_cancellation_error(error: str) -> bool:return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)+# opensearch-py renders transport failures as "ClassName(<status>, ...)". For the+# by-query APIs (_update_by_query / _delete_by_query) the response carries no+# top-level "error" key, so Connection._raise_error falls back to using the whole+# JSON response body as the exception message — several hundred characters of+# payload where a reason string would normally be.+#+# Classifying these early matters twice over: an infrastructure fault never+# reaches the UI as a raw exception repr, and the JSON body cannot trip the+# substring heuristics further down (a mapper_parsing_exception body contains+# "failed to parse", which would otherwise read as a corrupted file).+#+# ConnectionError / ConnectionTimeout are deliberately excluded: they render with+# a different __str__ and are already covered by _is_transient_connectivity_error.+_OPENSEARCH_TRANSPORT_ERROR_RE = re.compile(+ r"\b(?:ConflictError|NotFoundError|RequestError|TransportError"+ r"|AuthenticationException|AuthorizationException)\((\d{3})[,)]"+)+++def _opensearch_failure_metadata(error: str) -> dict | None:+ """Classify an opensearch-py transport failure surfaced as a task error.++ Returns None when the error is not an OpenSearch transport failure, so the+ caller falls through to the existing classification branches.+ """+ if not error:+ return None+ match = _OPENSEARCH_TRANSPORT_ERROR_RE.search(error)+ if not match:+ return None++ status = match.group(1)+ if status == "409":+ message = (+ "The search index was busy and this change conflicted with another "+ "update. Nothing is wrong with the file — retry ingestion."+ )+ else:+ message = (+ f"The search index rejected this document (OpenSearch error {status}). "+ "This is an infrastructure problem, not a problem with the file. "+ "Retry ingestion, and check the backend logs if it persists."+ )+ # RETRYABLE across the board: of the two available values, USER_ACTIONABLE+ # would be actively wrong — there is nothing the uploader can change about+ # their file to fix a search-index fault.+ return {+ "component": "opensearch",+ "failure_phase": "indexing",+ "user_facing_message": message,+ "actionable_by": "RETRYABLE",+ }++def _provider_credential_failure_metadata(error: str) -> dict | None:"""Classify invalid/revoked provider API key failures for ingestion toasts/UI."""if not error:@@ -1020,6 +1075,13 @@ class TaskService:"actionable_by": "USER_ACTIONABLE",}+ # Before any substring heuristic: an OpenSearch transport failure carries a+ # JSON payload whose contents would otherwise match the file-corruption and+ # "already exists" markers below.+ opensearch_meta = _opensearch_failure_metadata(error)+ if opensearch_meta:+ return opensearch_meta+if "incorrect password" in error.lower(): # for password protected pdf casesreturn {"component": "docling",diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.pyindex 697862547..8dd228896 100644--- a/tests/unit/test_task_service_get_task_status2.py+++ b/tests/unit/test_task_service_get_task_status2.py@@ -14,7 +14,7 @@ from unittest.mock import Mockimport pytestfrom models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask-from services.task_service import TaskService+from services.task_service import TaskService, _opensearch_failure_metadata# ---------------------------------------------------------------------------# Fixtures@@ -671,3 +671,151 @@ class TestGetTaskStatusRegression:for key in v1_file:assert key in v2_file, f"get_task_status2 file entry missing key: {key}"assert v1_file[key] == v2_file[key], f"file entry key '{key}' differs"+++# ---------------------------------------------------------------------------+# OpenSearch transport failures+# ---------------------------------------------------------------------------+++class TestOpenSearchFailureMetadata:+ def test_by_query_conflict_is_classified_as_retryable_opensearch_failure(self, task_service):+ # opensearch-py puts the entire by-query response body into the message,+ # because that response has no top-level "error" key to use instead.+ ft = _make_file_task(+ filename="logo.png",+ phase=IngestionPhase.LANGFLOW,+ error=(+ 'ConflictError(409, \'{"took":2719,"timed_out":false,"total":300,'+ '"updated":19,"version_conflicts":1,"failures":[{"index":"documents",'+ '"id":"20","cause":{"type":"version_conflict_engine_exception"}}]}\')'+ ),+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "opensearch"+ assert meta["failure_phase"] == "indexing"+ assert meta["actionable_by"] == "RETRYABLE"+ msg = meta["user_facing_message"]+ assert "retry ingestion" in msg.lower()+ # The raw payload must not reach the UI.+ assert "version_conflict_engine_exception" not in msg+ assert "{" not in msg++ def test_mapper_parsing_body_is_not_read_as_a_corrupt_file(self, task_service):+ # A 400 body contains the substring "failed to parse", which+ # _is_non_retryable_file_error would otherwise classify as a corrupt file.+ ft = _make_file_task(+ error="RequestError(400, 'mapper_parsing_exception', 'failed to parse field [x]')"+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "opensearch"+ assert "corrupt" not in meta["user_facing_message"].lower()++ def test_image_without_text_still_gets_the_ocr_guidance(self, task_service):+ ft = _make_file_task(+ filename="scan.png",+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error="No text content could be extracted from document",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+ assert "ocr" in meta["user_facing_message"].lower()++ def test_plain_errors_are_left_to_the_existing_branches(self, task_service):+ ft = _make_file_task(error="Docling conversion did not complete (timeout): 300s")+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+++class TestOpenSearchErrorStringContract:+ """Pin the opensearch-py rendering the classifier depends on.++ _opensearch_failure_metadata matches the exception's *string form*, which is+ built by opensearch-py's Connection._raise_error, not by us. If a client+ upgrade changes that rendering, the classifier silently stops matching and+ users start seeing raw payloads again. These tests fail loudly instead of+ letting that regress quietly.+ """++ def test_by_query_conflict_renders_body_as_the_message(self):+ from opensearchpy.connection.base import Connection+ from opensearchpy.exceptions import ConflictError++ # An _update_by_query 409 carries no top-level "error" key, so+ # _raise_error falls back to using the whole response body as the message.+ body = (+ '{"took":12,"timed_out":false,"total":3,"updated":1,'+ '"version_conflicts":1,"failures":[{"index":"documents","id":"7",'+ '"cause":{"type":"version_conflict_engine_exception"}}]}'+ )+ with pytest.raises(ConflictError) as excinfo:+ Connection()._raise_error(409, body, "application/json")++ rendered = str(excinfo.value)+ assert rendered.startswith("ConflictError(409, '{")+ meta = _opensearch_failure_metadata(rendered)+ assert meta is not None+ assert meta["component"] == "opensearch"+ assert meta["actionable_by"] == "RETRYABLE"++ def test_single_document_conflict_renders_a_short_reason(self):+ from opensearchpy.connection.base import Connection+ from opensearchpy.exceptions import ConflictError++ # Single-document writes render differently — a reason string rather than+ # a JSON body — and must classify identically.+ body = (+ '{"error":{"root_cause":[{"type":"version_conflict_engine_exception",'+ '"reason":"[7]: version conflict"}],'+ '"type":"version_conflict_engine_exception"},"status":409}'+ )+ with pytest.raises(ConflictError) as excinfo:+ Connection()._raise_error(409, body, "application/json")++ rendered = str(excinfo.value)+ assert rendered.startswith("ConflictError(409, 'version_conflict_engine_exception'")+ meta = _opensearch_failure_metadata(rendered)+ assert meta is not None+ assert meta["component"] == "opensearch"++ def test_non_opensearch_text_is_not_classified(self):+ for text in (+ "",+ "Docling conversion did not complete (timeout): polling exceeded 300s",+ "the sync hit a ConflictError while running", # prose, no (status)+ ):+ assert _opensearch_failure_metadata(text) is None+++class TestGetTaskStatus2OpenSearchFailure:+ def test_conflict_surfaces_as_structured_metadata(self, task_service):+ ft = _make_file_task(+ filename="logo.png",+ phase=IngestionPhase.LANGFLOW,+ error=(+ 'ConflictError(409, \'{"took":9,"total":3,"updated":1,'+ '"version_conflicts":1,"failures":[{"index":"documents","id":"7"}]}\')'+ ),+ )+ task = _make_upload_task("os409", {"logo.png": ft})+ task.status = TaskStatus.FAILED+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ file_entry = task_service.get_task_status2("user1", "os409")["files"]["logo.png"]++ assert file_entry["status"] == "failed"+ assert file_entry["component"] == "opensearch"+ assert file_entry["failure_phase"] == "indexing"+ assert file_entry["actionable_by"] == "RETRYABLE"+ # The payload must not reach the user as the displayed message...+ assert "{" not in file_entry["user_facing_message"]+ assert "version_conflicts" not in file_entry["user_facing_message"]+ # ...but the raw error stays on the entry, which is the only way to+ # diagnose which OpenSearch operation actually failed.+ assert file_entry["error"].startswith("ConflictError(409, '{")