OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: change docling manager to use updated docling, change docling component to raise errors (#1686)
Lucas Oliveira4e98122bd81c
diff --git a/flows/components/docling_remote.py b/flows/components/docling_remote.pyindex 177f3d455..a53699f1e 100644--- a/flows/components/docling_remote.py+++ b/flows/components/docling_remote.py@@ -4,7 +4,7 @@ import base64import jsonimport timefrom concurrent.futures import Future, ThreadPoolExecutor-from pathlib import Path+from pathlib import Path # noqa: TC003from typing import Anyimport httpx@@ -14,9 +14,7 @@ from pydantic import ValidationErrorfrom lfx.base.data import BaseFileComponentfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInputfrom lfx.inputs.inputs import FloatInput-from lfx.schema import Data-from lfx.schema.dotdict import dotdict-from lfx.schema.message import Message+from lfx.schema import Data, dotdictfrom lfx.utils.util import transform_localhost_url@@ -255,6 +253,20 @@ class DoclingRemoteComponent(BaseFileComponent):result_resp.raise_for_status()result = result_resp.json()+ if result.get("status") == "failure" or result.get("errors"):+ errors = result.get("errors", [])+ err_msg_list = []+ for err in errors:+ if isinstance(err, dict) and "error_message" in err:+ err_msg_list.append(err["error_message"])+ elif isinstance(err, str):+ err_msg_list.append(err)++ err_details = "; ".join(err_msg_list) if err_msg_list else "Unknown Docling processing error"++ msg = f"Docling processing failed: {err_details}"+ raise ValueError(msg)+if "json_content" not in result["document"] or result["document"]["json_content"] is None:self.log("No JSON DoclingDocument found in the result.")return Nonediff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex 79e309e64..6a68aabad 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -828,7 +828,7 @@],"frozen": false,"icon": "braces",- "last_updated": "2026-05-08T19:20:38.334Z",+ "last_updated": "2026-05-26T19:04:35.148Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {},@@ -878,7 +878,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "a451a847-ad7a-442f-ab14-569b43d392bd"+ "value": "d61134a0-3f8e-4e89-9165-ce13f2a14ac6"},"_type": "Component","code": {@@ -1336,10 +1336,9 @@],"frozen": false,"icon": "Docling",- "last_updated": "2026-05-08T19:20:27.016Z","legacy": false,"metadata": {- "code_hash": "3a18033f9e4f",+ "code_hash": "dcf9aeedd059","dependencies": {"dependencies": [{@@ -1348,7 +1347,7 @@},{"name": "docling_core",- "version": "2.73.0"+ "version": "2.74.1"},{"name": "pydantic",@@ -1356,7 +1355,7 @@},{"name": "lfx",- "version": "0.4.0"+ "version": "0.4.3"}],"total_dependencies": 4@@ -1387,12 +1386,6 @@],"pinned": false,"template": {- "_frontend_node_flow_id": {- "value": "5488df7c-b93f-4f87-a446-b67028bc0813"- },- "_frontend_node_folder_id": {- "value": "a451a847-ad7a-442f-ab14-569b43d392bd"- },"_type": "Component","api_headers": {"_input_type": "TableInput",@@ -1486,7 +1479,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self._process_headers()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"+ "value": "from __future__ import annotations\n\nimport base64\nimport json\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path # noqa: TC003\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput, TableInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data, dotdict\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = (\n \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n )\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n StrInput(\n name=\"task_id\",\n display_name=\"Task ID\",\n info=(\n \"Optional task ID from a previous Docling Serve upload. \"\n \"If provided, file input is ignored and the component polls for this task's results.\"\n ),\n required=False,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n TableInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional headers required for connecting to Docling Serve.\"),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"string\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"load_from_db\": True,\n \"type\": \"string\",\n \"description\": \"Value of the header\",\n },\n ],\n value=[],\n real_time_refresh=True,\n input_types=[\"Data\", \"JSON\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def _process_headers(self) -> dict[str, str]:\n \"\"\"Process the headers input into a valid dictionary.\"\"\"\n if not self.api_headers:\n return {}\n\n component_headers_dict = {}\n # TableInput normalizes to list\n items = self.api_headers if isinstance(self.api_headers, list) else [self.api_headers]\n\n for item in items:\n if not item:\n continue\n\n # Case 1: Data object\n if hasattr(item, \"data\") and isinstance(item.data, dict):\n data = item.data\n if \"key\" in data and \"value\" in data:\n component_headers_dict[str(data[\"key\"])] = str(data[\"value\"])\n else:\n # Fallback: merge all keys from Data object\n for k, v in data.items():\n if k not in (\"text_key\", \"default_value\"):\n component_headers_dict[str(k)] = str(v)\n\n # Case 2: Dictionary (Table row)\n elif isinstance(item, dict):\n if \"key\" in item and \"value\" in item:\n component_headers_dict[str(item[\"key\"])] = str(item[\"value\"])\n else:\n # Fallback: merge all keys\n for k, v in item.items():\n component_headers_dict[str(k)] = str(v)\n\n # Case 3: Message object\n elif hasattr(item, \"text\") and isinstance(item.text, str):\n try:\n parsed = json.loads(item.text)\n if isinstance(parsed, dict):\n for k, v in parsed.items():\n component_headers_dict[str(k)] = str(v)\n except json.JSONDecodeError:\n pass\n\n return component_headers_dict\n\n def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n if field_name == \"api_headers\":\n if isinstance(field_value, dict):\n # If it's a dict, convert to list of {key, value} pairs for TableInput\n # This handles migration from NestedDictInput to TableInput\n new_value = [{\"key\": k, \"value\": v} for k, v in field_value.items()]\n build_config[\"api_headers\"][\"value\"] = new_value\n return build_config\n if field_value is None:\n build_config[\"api_headers\"][\"value\"] = []\n return build_config\n\n # Default behavior\n return super().update_build_config(build_config, field_value, field_name)\n\n def _poll_and_fetch_result(\n self, client: httpx.Client, base_url: str, task_id: str, file_path: str | None = None\n ) -> Data | None:\n \"\"\"Poll for task completion and fetch the result.\n\n Args:\n client: The HTTP client to use for requests.\n base_url: The base URL of the Docling Serve API.\n task_id: The task ID to poll for.\n file_path: Optional file path to include in the result data.\n\n Returns:\n Data object with the DoclingDocument, or None if processing failed.\n \"\"\"\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n response.raise_for_status()\n task = response.json()\n\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task_id}\")\n\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(\n f\"The status requests got a http response {response.status_code} too many times.\"\n )\n return None\n continue\n\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task_id}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if result.get(\"status\") == \"failure\" or result.get(\"errors\"):\n errors = result.get(\"errors\", [])\n err_msg_list = []\n for err in errors:\n if isinstance(err, dict) and \"error_message\" in err:\n err_msg_list.append(err[\"error_message\"])\n elif isinstance(err, str):\n err_msg_list.append(err)\n\n err_details = \"; \".join(err_msg_list) if err_msg_list else \"Unknown Docling processing error\"\n\n msg = f\"Docling processing failed: {err_details}\"\n raise ValueError(msg)\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n data_dict: dict[str, Any] = {\"doc\": doc}\n if file_path:\n data_dict[\"file_path\"] = file_path\n return Data(data=data_dict)\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n def _process_task_id(self) -> list[Data]:\n \"\"\"Process an existing task by polling for status and retrieving results.\n\n Returns:\n List containing the result Data object, or empty list if processing failed.\n \"\"\"\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n with httpx.Client(headers=self._process_headers()) as client:\n result = self._poll_and_fetch_result(client, base_url, self.task_id)\n return [result] if result else []\n\n def load_files_base(self) -> list[Data]:\n \"\"\"Load and process files, or poll an existing task if task_id is provided.\n\n Returns:\n list[Data]: Parsed data from the processed files or task.\n \"\"\"\n if self.task_id:\n return self._process_task_id()\n return super().load_files_base()\n\n def process_files(\n self, file_list: list[BaseFileComponent.BaseFile]\n ) -> list[BaseFileComponent.BaseFile]:\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(\n client: httpx.Client, file_path: Path, options: dict[str, Any]\n ) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [\n {\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}\n ],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n return self._poll_and_fetch_result(client, base_url, task[\"task_id\"], str(file_path))\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self._process_headers()) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append(\n (i, executor.submit(_convert_document, client, file.path, docling_options))\n )\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n"},"delete_server_file_after_processing": {"_input_type": "BoolInput",@@ -1607,7 +1600,6 @@"type": "bool","value": true},- "is_refresh": false,"max_concurrency": {"_input_type": "IntInput","advanced": true,@@ -1788,8 +1780,8 @@"width": 192},"position": {- "x": 4.296782621507347,- "y": 1764.5830512766954+ "x": -41.950234273170665,+ "y": 1766.6851884082719},"selected": false,"type": "genericNode"@@ -4680,7 +4672,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "a451a847-ad7a-442f-ab14-569b43d392bd"+ "value": "d61134a0-3f8e-4e89-9165-ce13f2a14ac6"},"_type": "Component","api_base": {@@ -4949,96 +4941,6 @@},"name": "text-embedding-ada-002","provider": "OpenAI"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/granite-embedding-278m-multilingual",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-125m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-30m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "intfloat/multilingual-e5-large",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "sentence-transformers/all-minilm-l6-v2",- "provider": "IBM WatsonX"}],"override_skip": false,@@ -6522,7 +6424,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "a451a847-ad7a-442f-ab14-569b43d392bd"+ "value": "d61134a0-3f8e-4e89-9165-ce13f2a14ac6"},"_type": "Component","api_base": {@@ -6791,96 +6693,6 @@},"name": "text-embedding-ada-002","provider": "OpenAI"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/granite-embedding-278m-multilingual",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-125m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-30m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "intfloat/multilingual-e5-large",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "sentence-transformers/all-minilm-l6-v2",- "provider": "IBM WatsonX"}],"override_skip": false,@@ -7140,7 +6952,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "a451a847-ad7a-442f-ab14-569b43d392bd"+ "value": "d61134a0-3f8e-4e89-9165-ce13f2a14ac6"},"_type": "Component","api_base": {@@ -7408,96 +7220,6 @@},"name": "text-embedding-ada-002","provider": "OpenAI"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/granite-embedding-278m-multilingual",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-125m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "ibm/slate-30m-english-rtrvr-v2",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "intfloat/multilingual-e5-large",- "provider": "IBM WatsonX"- },- {- "category": "IBM WatsonX",- "icon": "WatsonxAI",- "metadata": {- "embedding_class": "WatsonxEmbeddings",- "model_type": "embeddings",- "param_mapping": {- "api_key": "apikey",- "model_id": "model_id",- "project_id": "project_id",- "request_timeout": "request_timeout",- "space_id": "space_id",- "url": "url"- }- },- "name": "sentence-transformers/all-minilm-l6-v2",- "provider": "IBM WatsonX"}],"override_skip": false,@@ -7682,16 +7404,16 @@}],"viewport": {- "x": 193.22171349647556,- "y": -404.7149184337984,- "zoom": 0.5164469534707714+ "x": 239.96894590419038,+ "y": -373.3174538705754,+ "zoom": 0.475706358533402}},"description": "Load your data for chat context with Retrieval Augmented Generation.","endpoint_name": null,"id": "5488df7c-b93f-4f87-a446-b67028bc0813","is_component": false,- "last_tested_version": "1.9.0",+ "last_tested_version": "1.9.3","locked": true,"name": "OpenSearch Ingestion Flow","tags": [@@ -7700,4 +7422,4 @@"rag","q-a"]-}\ No newline at end of file+}diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.pyindex bdf1b27e2..903a480a1 100644--- a/src/tui/managers/docling_manager.py+++ b/src/tui/managers/docling_manager.py@@ -7,7 +7,9 @@ import sysimport tempfileimport threadingimport time-from typing import Optional, Tuple, Dict, Any, List, AsyncIterator+from collections.abc import AsyncIterator+from typing import Any+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -29,7 +31,7 @@ class DoclingManager:if self._initialized:return- self._process: Optional[subprocess.Popen] = None+ self._process: subprocess.Popen | None = Noneself._port = 5001# Bind to all interfaces by default (can be overridden with DOCLING_BIND_HOST env var)self._host = os.getenv("DOCLING_BIND_HOST", "0.0.0.0")@@ -47,7 +49,7 @@ class DoclingManager:self._log_file_path = self._tui_dir / "docling-serve.log"# Log storage - simplified, no queue- self._log_buffer: List[str] = []+ self._log_buffer: list[str] = []self._max_log_lines = 1000self._log_lock = threading.Lock() # Thread-safe access to log buffer@@ -71,7 +73,7 @@ class DoclingManager:except Exception as e:self._add_log_entry(f"Failed to save PID file: {e}")- def _load_pid(self) -> Optional[int]:+ def _load_pid(self) -> int | None:"""Load the process PID from file."""try:if self._pid_file.exists():@@ -105,9 +107,7 @@ class DoclingManager:pid = self._load_pid()if pid is not None:if self._is_process_running(pid):- self._add_log_entry(- f"Recovered existing docling-serve process (PID: {pid})"- )+ self._add_log_entry(f"Recovered existing docling-serve process (PID: {pid})")# Mark as external process since we didn't start it in this sessionself._external_process = Trueself._running = True@@ -150,7 +150,7 @@ class DoclingManager:self._external_process = Falsereturn False- def check_port_available(self) -> tuple[bool, Optional[str]]:+ def check_port_available(self) -> tuple[bool, str | None]:"""Check if the native service port is available.Returns:@@ -173,7 +173,7 @@ class DoclingManager:# If we can't check, assume it's availablereturn True, None- def get_status(self) -> Dict[str, Any]:+ def get_status(self) -> dict[str, Any]:"""Get current status of docling serve."""# Check for starting state firstif self._starting:@@ -230,10 +230,10 @@ class DoclingManager:enable_ui: bool = False,workers: int | None = None,timeout: int = 10,- ) -> Tuple[bool, str]:+ ) -> tuple[bool, str]:"""Start docling serve as external process.-+Args:port: Port to listen on (default: 5001)host: Host to bind to (default: from env or 0.0.0.0)@@ -280,24 +280,33 @@ class DoclingManager:# override file is needed to *replace* (not merely supplement)# opencv-python, because docling-ibm-models hard-depends on it and# the resolver would install both packages otherwise.- override_path: Optional[str] = None+ override_path: str | None = Noneif sys.platform != "darwin":- fd, override_path = tempfile.mkstemp(- suffix=".txt", prefix="docling_cv_override_"- )+ fd, override_path = tempfile.mkstemp(suffix=".txt", prefix="docling_cv_override_")with os.fdopen(fd, "w") as f:f.write('opencv-python ; python_version < "0"\n')try:- docling_extras = "ocrmac,easyocr,rapidocr,vlm" if sys.platform == "darwin" else "easyocr,rapidocr,vlm"+ docling_extras = (+ "ocrmac,easyocr,rapidocr,vlm"+ if sys.platform == "darwin"+ else "easyocr,rapidocr,vlm"+ )cmd = ["uvx",- "--from", "docling-serve[ui]==1.15.1",- "--with", "onnxruntime",- "--with", "easyocr",- "--with", f"docling[{docling_extras}]",- "--with", "docling-core==2.71.0",+ "--from",+ "docling-serve[ui]==1.20.0",+ "--with",+ "onnxruntime",+ "--with",+ "easyocr",+ "--with",+ f"docling[{docling_extras}]",+ "--with",+ "docling-core==2.77.1",+ "--with",+ "transformers>=5.8.1,<5.9.0",]if override_path:cmd += ["--override", override_path, "--with", "opencv-python-headless"]@@ -343,9 +352,7 @@ class DoclingManager:self._start_output_capture()# Wait for the process to start and begin listening- self._add_log_entry(- f"Waiting up to {timeout}s for docling-serve to start listening..."- )+ self._add_log_entry(f"Waiting up to {timeout}s for docling-serve to start listening...")for i in range(timeout):await asyncio.sleep(1.0)@@ -369,8 +376,8 @@ class DoclingManager:)self._starting = Falsebreak- except:- pass+ except Exception as e:+ self._add_log_entry(f"Error waiting for docling-serve: {e}")if (i + 1) % 10 == 0:self._add_log_entry(f"Waiting for startup... ({i + 1}/{timeout}s)")@@ -382,9 +389,7 @@ class DoclingManager:except OSError:pass- self._add_log_entry(- f"Process PID: {self._process.pid}, Poll: {self._process.poll()}"- )+ self._add_log_entry(f"Process PID: {self._process.pid}, Poll: {self._process.poll()}")if self._process.poll() is not None:# Process already exited - get return code and any output@@ -447,7 +452,7 @@ class DoclingManager:self._add_log_entry("Starting log file capture thread")try:- with open(self._log_file_path, "r") as f:+ with open(self._log_file_path) as f:while self._running and self._process and self._process.poll() is None:line = f.readline()if line:@@ -466,7 +471,7 @@ class DoclingManager:self._add_log_entry("Log file capture thread started")- async def stop(self) -> Tuple[bool, str]:+ async def stop(self) -> tuple[bool, str]:"""Stop docling serve."""if not self.is_running():return False, "Docling serve is not running"@@ -497,18 +502,14 @@ class DoclingManager:# This is a process we recovered from PID filepid_to_stop = self._load_pid()if pid_to_stop and self._is_process_running(pid_to_stop):- self._add_log_entry(- f"Stopping process from PID file (PID: {pid_to_stop})"- )+ self._add_log_entry(f"Stopping process from PID file (PID: {pid_to_stop})")try:os.kill(pid_to_stop, 15) # SIGTERM# Wait a bit for graceful shutdownawait asyncio.sleep(2)if self._is_process_running(pid_to_stop):# Still running, force kill- self._add_log_entry(- f"Force killing process (PID: {pid_to_stop})"- )+ self._add_log_entry(f"Force killing process (PID: {pid_to_stop})")os.kill(pid_to_stop, 9) # SIGKILLexcept Exception as e:self._add_log_entry(f"Error stopping external process: {e}")@@ -533,10 +534,10 @@ class DoclingManager:async def restart(self,- port: Optional[int] = None,- host: Optional[str] = None,+ port: int | None = None,+ host: str | None = None,enable_ui: bool = False,- ) -> Tuple[bool, str]:+ ) -> tuple[bool, str]:"""Restart docling serve."""# Use current settings if not specifiedif port is None:@@ -560,7 +561,7 @@ class DoclingManager:"""Add a manual log entry - useful for debugging."""self._add_log_entry(f"MANUAL: {message}")- def get_logs(self, lines: int = 50) -> Tuple[bool, str]:+ def get_logs(self, lines: int = 50) -> tuple[bool, str]:"""Get logs from the docling-serve process."""if self.is_running():with self._log_lock:@@ -577,10 +578,6 @@ class DoclingManager:async def follow_logs(self) -> AsyncIterator[str]:"""Follow logs from the docling-serve process in real-time."""- # First yield status message and any existing logs- display_host = "localhost" if self._host == "0.0.0.0" else self._host- status_msg = f"Docling serve is running on http://{display_host}:{self._port}"-with self._log_lock:if self._log_buffer:yield "\n".join(self._log_buffer)