Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

support images in non langflow ingestion (#2172)

Mike Fortmana38d14517997
diff --git a/src/utils/document_processing.py b/src/utils/document_processing.py
index e35b8a353..af2b65706 100644
--- a/src/utils/document_processing.py
+++ b/src/utils/document_processing.py
@@ -131,6 +131,34 @@ def extract_relevant(doc_dict: dict) -> dict:
}
)
+ # 3) process picture descriptions (VLM-generated annotations)
+ # Docling stores these under pictures[].annotations with kind == "description".
+ # Without this, image-only documents (no text layer) yield zero chunks on the
+ # non-Langflow path, which surfaces as a misleading "corrupted or invalid" error.
+ # The Langflow path already captures them via Docling's export_to_markdown.
+ for p_idx, picture in enumerate(doc_dict.get("pictures", [])):
+ prov = picture.get("prov", [])
+ page_no = prov[0].get("page_no") if prov else None
+ if page_no is None:
+ page_no = 1
+
+ descriptions = [
+ ann.get("text", "").strip()
+ for ann in picture.get("annotations", [])
+ if ann.get("kind") == "description" and ann.get("text", "").strip()
+ ]
+ if not descriptions:
+ continue
+
+ chunks.append(
+ {
+ "page": page_no,
+ "type": "picture",
+ "picture_index": p_idx,
+ "text": "\n".join(descriptions),
+ }
+ )
+
return {
"id": origin.get("binary_hash"),
"filename": origin.get("filename"),
diff --git a/tests/unit/test_extract_relevant_pictures.py b/tests/unit/test_extract_relevant_pictures.py
new file mode 100644
index 000000000..2b3b278a5
--- /dev/null
+++ b/tests/unit/test_extract_relevant_pictures.py
@@ -0,0 +1,96 @@
+"""Tests for extract_relevant picture-description (VLM annotation) handling.
+
+Regression coverage for image-only documents on the non-Langflow ingestion
+path: Docling stores VLM captions under pictures[].annotations, and dropping
+them left such documents with zero chunks (surfaced as "corrupted or invalid").
+"""
+
+from src.utils.document_processing import extract_relevant
+
+
+def _description(text: str) -> dict:
+ return {"kind": "description", "text": text, "provenance": "vlm-model"}
+
+
+def test_image_only_doc_yields_picture_chunk():
+ """A document with no text layer still produces a chunk from its caption."""
+ doc = {
+ "origin": {"binary_hash": "h1", "filename": "cat.pdf", "mimetype": "application/pdf"},
+ "texts": [],
+ "tables": [],
+ "pictures": [
+ {"prov": [{"page_no": 1}], "annotations": [_description("a cat on a mat")]},
+ ],
+ }
+ result = extract_relevant(doc)
+ picture_chunks = [c for c in result["chunks"] if c["type"] == "picture"]
+ assert len(picture_chunks) == 1
+ assert picture_chunks[0]["text"] == "a cat on a mat"
+ assert picture_chunks[0]["page"] == 1
+ assert picture_chunks[0]["picture_index"] == 0
+
+
+def test_multiple_descriptions_joined():
+ doc = {
+ "origin": {},
+ "pictures": [
+ {
+ "prov": [{"page_no": 3}],
+ "annotations": [_description("line one"), _description("line two")],
+ }
+ ],
+ }
+ chunk = extract_relevant(doc)["chunks"][0]
+ assert chunk["text"] == "line one\nline two"
+ assert chunk["page"] == 3
+
+
+def test_classification_only_annotation_is_skipped():
+ """Non-description annotations (e.g. classification) carry no caption text."""
+ doc = {
+ "origin": {},
+ "pictures": [
+ {
+ "prov": [{"page_no": 1}],
+ "annotations": [
+ {
+ "kind": "classification",
+ "provenance": "cls",
+ "predicted_classes": [{"class_name": "photo", "confidence": 0.9}],
+ }
+ ],
+ }
+ ],
+ }
+ assert extract_relevant(doc)["chunks"] == []
+
+
+def test_empty_or_whitespace_description_skipped():
+ doc = {
+ "origin": {},
+ "pictures": [
+ {"prov": [{"page_no": 1}], "annotations": [_description(" ")]},
+ {"prov": [{"page_no": 2}], "annotations": []},
+ ],
+ }
+ assert extract_relevant(doc)["chunks"] == []
+
+
+def test_missing_prov_defaults_to_page_one():
+ doc = {"origin": {}, "pictures": [{"annotations": [_description("no prov")]}]}
+ chunk = extract_relevant(doc)["chunks"][0]
+ assert chunk["page"] == 1
+ assert chunk["text"] == "no prov"
+
+
+def test_text_and_picture_chunks_coexist():
+ doc = {
+ "origin": {},
+ "texts": [{"prov": [{"page_no": 1}], "text": "body text"}],
+ "pictures": [{"prov": [{"page_no": 1}], "annotations": [_description("a diagram")]}],
+ }
+ result = extract_relevant(doc)
+ types = {c["type"] for c in result["chunks"]}
+ assert types == {"text", "picture"}
+ picture_chunk = next(c for c in result["chunks"] if c["type"] == "picture")
+ assert picture_chunk["text"] == "a diagram"