OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: enhance error handling for document processing and add OCR requirement check for image files (#1859) (#2017)
Rico Furtado4c6acd8551dd
diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.tsindex 221151edc..f52110090 100644--- a/sdks/typescript/tests/integration.test.ts+++ b/sdks/typescript/tests/integration.test.ts@@ -363,11 +363,33 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {});it("should delete a document", async () => {+ // Use a uniquely named file so this test doesn't inherit chunks left in+ // the index by the two ingest tests above (which share testFilePath /+ // "sdk_test_doc.md"). Otherwise a zero-successful-files re-ingest here can+ // still find the earlier document, making delete succeed when the+ // else-branch expects a no-op — i.e. ingest's reported successful_files+ // would no longer reflect whether THIS document exists in the index.+ const deleteDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-del-"));+ const deleteFilePath = path.join(+ deleteDir,+ `sdk_delete_doc_${Date.now()}.md`+ );+ fs.writeFileSync(+ deleteFilePath,+ "# SDK Delete Test Document\n\n" ++ "This document tests document deletion via the OpenRAG TypeScript SDK.\n" ++ "It contains unique content about teal dolphins swimming.\n"+ );+// First ingest (wait for completion)- const ingestResult = await client.documents.ingest({ filePath: testFilePath });+ const ingestResult = await client.documents.ingest({+ filePath: deleteFilePath,+ });// Then delete- const result = await client.documents.delete(path.basename(testFilePath));+ const result = await client.documents.delete(+ path.basename(deleteFilePath)+ );// If ingestion produced indexed chunks, delete should succeed.// In unstable flow environments, ingestion can complete with zero successful files.diff --git a/src/models/processors.py b/src/models/processors.pyindex c767a2b61..25886bd55 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -74,6 +74,8 @@ class TaskProcessor:file_hash: str,opensearch_client,on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",+ *,+ wait_for_visibility: bool = False,) -> bool:"""Check if a document with the given hash already exists in OpenSearch.@@ -86,6 +88,13 @@ class TaskProcessor:* ``"assume_exists"`` (post-ingestion verification callers): aninfrastructure error must not fail a file that Langflow alreadyreported as ingested.++ When ``wait_for_visibility`` is True, an empty result is retried a few+ times with backoff before concluding the document is absent. This is for+ post-ingest verification: chunks that were just written may not be+ searchable yet within OpenSearch's near-real-time refresh window+ (default ~1s), and the user-scoped client cannot force an+ ``indices:admin/refresh`` (it lacks the privilege)."""max_retries = 3retry_delay = 1.0@@ -101,7 +110,15 @@ class TaskProcessor:},)hits = response.get("hits", {}).get("hits", [])- return bool(hits)+ if hits:+ return True+ # No hits. For post-ingest verification, the document may not be+ # visible yet within the near-real-time refresh window — retry.+ if wait_for_visibility and attempt < max_retries - 1:+ await asyncio.sleep(retry_delay)+ retry_delay *= 2+ continue+ return Falseexcept (TimeoutError, Exception) as e:if attempt == max_retries - 1:logger.error(@@ -139,10 +156,19 @@ class TaskProcessor:self,filename: str,opensearch_client,+ *,+ wait_for_visibility: bool = False,) -> bool:"""Check if a document with the given filename already exists in OpenSearch.Returns True if any chunks with this filename exist.++ When ``wait_for_visibility`` is True, an empty result is retried a few+ times with backoff before concluding the document is absent. This is for+ post-ingest verification: chunks that were just written may not be+ searchable yet within OpenSearch's near-real-time refresh window+ (default ~1s), and the user-scoped client cannot force an+ ``indices:admin/refresh`` (it lacks the privilege)."""max_retries = 3retry_delay = 1.0@@ -174,6 +200,14 @@ class TaskProcessor:# re-query it on future retries.pending_candidates.pop(i)continue+ # All aliases checked with no hits. For post-ingest verification,+ # the document may not be visible yet within the near-real-time+ # refresh window — re-check every alias after a short delay.+ if wait_for_visibility and attempt < max_retries - 1:+ pending_candidates = list(candidate_filenames)+ await asyncio.sleep(retry_delay)+ retry_delay *= 2+ continuereturn Falseexcept (TimeoutError, Exception) as e:@@ -1057,10 +1091,14 @@ class ConnectorFileProcessor(TaskProcessor):# Langflow returns "success" even when no text was extracted# (e.g. image files without OCR). Verify the document actually# landed in OpenSearch before declaring success.+ # wait_for_visibility polls on an empty result to ride out+ # OpenSearch's ~1s near-real-time window (the user-scoped+ # client cannot force an indices:admin/refresh — it 403s).if not await self.check_document_exists(document.id,_verification_client(opensearch_client),on_error="assume_exists",+ wait_for_visibility=True,):result = {"status": "error",@@ -1367,13 +1405,22 @@ class LangflowFileProcessor(TaskProcessor):file_task=file_task,)- # Langflow returns "success" even when no text was extracted.- # Verify the document actually landed in OpenSearch.- file_hash = hash_id(item)- if not await self.check_document_exists(- file_hash,+ # Langflow returns "success" even when no text was extracted+ # (e.g. image files without OCR). Verify the document actually+ # landed in OpenSearch before declaring success. We key off the+ # filename — the identifier this path already uses for dedup and+ # delete (see check_filename_exists / delete_document_by_filename+ # above) — because Langflow assigns its own document_id here, so+ # hash_id(item) is not stored as document_id.+ #+ # wait_for_visibility polls on an empty result so the just-written+ # chunks become visible within OpenSearch's near-real-time refresh+ # window. We cannot force a refresh here: the user-scoped client+ # lacks the indices:admin/refresh privilege (it 403s).+ if not await self.check_filename_exists(+ original_filename,_verification_client(opensearch_client),- on_error="assume_exists",+ wait_for_visibility=True,):file_task.status = TaskStatus.FAILEDfile_task.error = "No text content could be extracted from document"