OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: universal file names (#1982)
Edwin Joseefd5f677c6d1
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex fd67e465e..f59f7a69d 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -35,7 +35,7 @@ env:AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}# Single-node test cluster never reaches the multi-node readiness counts- OPENSEARCH_NODE_COUNT_CHECK: "false"+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false"jobs:changes:diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex cd9e9d72e..2456116c4 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -32,35 +32,6 @@"target": "ExportDoclingDocument-zZdRg","targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œJSONœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"},- {- "animated": false,- "className": "",- "data": {- "sourceHandle": {- "dataType": "ExportDoclingDocument",- "id": "ExportDoclingDocument-zZdRg",- "name": "dataframe",- "output_types": [- "Table"- ]- },- "targetHandle": {- "fieldName": "df",- "id": "DataFrameOperations-1BWXB",- "inputTypes": [- "DataFrame",- "Table"- ],- "type": "other"- }- },- "id": "reactflow__edge-ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",- "selected": false,- "source": "ExportDoclingDocument-zZdRg",- "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",- "target": "DataFrameOperations-1BWXB",- "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"- },{"animated": false,"className": "",@@ -93,35 +64,6 @@"target": "SplitText-QIKhg","targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œJSONœ,œDataFrameœ,œTableœ,œMessageœ],œtypeœ:œotherœ}"},- {- "animated": false,- "className": "",- "data": {- "sourceHandle": {- "dataType": "DataFrameOperations",- "id": "DataFrameOperations-1BWXB",- "name": "output",- "output_types": [- "Table"- ]- },- "targetHandle": {- "fieldName": "df",- "id": "DataFrameOperations-9vMrp",- "inputTypes": [- "DataFrame",- "Table"- ],- "type": "other"- }- },- "id": "reactflow__edge-DataFrameOperations-1BWXB{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-9vMrp{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",- "selected": false,- "source": "DataFrameOperations-1BWXB",- "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œTableœ]}",- "target": "DataFrameOperations-9vMrp",- "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"- },{"animated": false,"className": "",@@ -264,6 +206,35 @@"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}","target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4","targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"+ },+ {+ "animated": false,+ "className": "",+ "data": {+ "sourceHandle": {+ "dataType": "ExportDoclingDocument",+ "id": "ExportDoclingDocument-zZdRg",+ "name": "dataframe",+ "output_types": [+ "Table"+ ]+ },+ "targetHandle": {+ "fieldName": "df",+ "id": "DataFrameOperations-9vMrp",+ "inputTypes": [+ "DataFrame",+ "Table"+ ],+ "type": "other"+ }+ },+ "id": "xy-edge__ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-9vMrp{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",+ "selected": false,+ "source": "ExportDoclingDocument-zZdRg",+ "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",+ "target": "DataFrameOperations-9vMrp",+ "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"}],"nodes": [@@ -535,8 +506,8 @@"width": 320},"position": {- "x": 1358.8096961625195,- "y": 1779.155571003572+ "x": 1104.7822073166224,+ "y": 1766.8724817385216},"positionAbsolute": {"x": 1683.4543896546102,@@ -1156,562 +1127,57 @@"dynamic": true,"fileTypes": [],"file_path": "",- "info": "",- "list": false,- "load_from_db": false,- "multiline": true,- "name": "code",- "password": false,- "placeholder": "",- "required": true,- "show": true,- "title_case": false,- "type": "code",- "value": "from typing import Any\n\nfrom docling_core.types.doc import ImageRefMode\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ExportDoclingDocumentComponent(Component):\n display_name: str = \"Export DoclingDocument\"\n description: str = \"Export DoclingDocument to markdown, html or other formats.\"\n documentation = \"https://docling-project.github.io/docling/\"\n icon = \"Docling\"\n name = \"ExportDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"JSON or Table\",\n info=\"The data with documents to export.\",\n input_types=[\"Data\", \"JSON\", \"DataFrame\", \"Table\"],\n required=True,\n ),\n DropdownInput(\n name=\"export_format\",\n display_name=\"Export format\",\n options=[\"Markdown\", \"HTML\", \"Plaintext\", \"DocTags\"],\n info=\"Select the export format to convert the input.\",\n value=\"Markdown\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"image_mode\",\n display_name=\"Image export mode\",\n options=[\"placeholder\", \"embedded\"],\n info=(\n \"Specify how images are exported in the output. Placeholder will replace the images with a string, \"\n \"whereas Embedded will include them as base64 encoded images.\"\n ),\n value=\"placeholder\",\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder betweek pages in the markdown output.\",\n value=\"\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Exported data\", name=\"data\", method=\"export_document\"),\n Output(display_name=\"Table\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"export_format\" and field_value == \"Markdown\":\n build_config[\"md_image_placeholder\"][\"show\"] = True\n build_config[\"md_page_break_placeholder\"][\"show\"] = True\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value == \"HTML\":\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value in {\"Plaintext\", \"DocTags\"}:\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = False\n\n return build_config\n\n def export_document(self) -> list[Data]:\n documents, warning = extract_docling_documents(self.data_inputs, self.doc_key)\n if warning:\n self.status = warning\n\n results: list[Data] = []\n try:\n image_mode = ImageRefMode(self.image_mode)\n for doc in documents:\n content = \"\"\n if self.export_format == \"Markdown\":\n content = doc.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n elif self.export_format == \"HTML\":\n content = doc.export_to_html(image_mode=image_mode)\n elif self.export_format == \"Plaintext\":\n content = doc.export_to_text()\n elif self.export_format == \"DocTags\":\n content = doc.export_to_doctags()\n\n # Preserve metadata from the DoclingDocument\n metadata: dict = {\"export_format\": self.export_format}\n if hasattr(doc, \"name\") and doc.name:\n metadata[\"name\"] = doc.name\n if hasattr(doc, \"origin\") and doc.origin is not None:\n if hasattr(doc.origin, \"filename\") and doc.origin.filename:\n metadata[\"filename\"] = doc.origin.filename\n if hasattr(doc.origin, \"binary_hash\") and doc.origin.binary_hash:\n metadata[\"document_id\"] = str(doc.origin.binary_hash)\n if hasattr(doc.origin, \"mimetype\") and doc.origin.mimetype:\n metadata[\"mimetype\"] = doc.origin.mimetype\n\n results.append(Data(text=content, data={\"text\": content, **metadata}))\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return results\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.export_document())\n"- },- "data_inputs": {- "_input_type": "HandleInput",- "advanced": false,- "display_name": "JSON or Table",- "dynamic": false,- "info": "The data with documents to export.",- "input_types": [- "Data",- "JSON",- "DataFrame",- "Table"- ],- "list": false,- "list_add_label": "Add More",- "name": "data_inputs",- "override_skip": false,- "placeholder": "",- "required": true,- "show": true,- "title_case": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "other",- "value": ""- },- "doc_key": {- "_input_type": "MessageTextInput",- "advanced": true,- "display_name": "Doc Key",- "dynamic": false,- "info": "The key to use for the DoclingDocument column.",- "input_types": [- "Message"- ],- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "doc_key",- "override_skip": false,- "placeholder": "",- "required": false,- "show": true,- "title_case": false,- "tool_mode": false,- "trace_as_input": true,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": "doc"- },- "export_format": {- "_input_type": "DropdownInput",- "advanced": false,- "combobox": false,- "dialog_inputs": {},- "display_name": "Export format",- "dynamic": false,- "external_options": {},- "info": "Select the export format to convert the input.",- "name": "export_format",- "options": [- "Markdown",- "HTML",- "Plaintext",- "DocTags"- ],- "options_metadata": [],- "override_skip": false,- "placeholder": "",- "real_time_refresh": true,- "required": false,- "show": true,- "title_case": false,- "toggle": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "str",- "value": "Markdown"- },- "image_mode": {- "_input_type": "DropdownInput",- "advanced": false,- "combobox": false,- "dialog_inputs": {},- "display_name": "Image export mode",- "dynamic": false,- "external_options": {},- "info": "Specify how images are exported in the output. Placeholder will replace the images with a string, whereas Embedded will include them as base64 encoded images.",- "name": "image_mode",- "options": [- "placeholder",- "embedded"- ],- "options_metadata": [],- "override_skip": false,- "placeholder": "",- "required": false,- "show": true,- "title_case": false,- "toggle": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "str",- "value": "placeholder"- },- "md_image_placeholder": {- "_input_type": "StrInput",- "advanced": true,- "display_name": "Image placeholder",- "dynamic": false,- "info": "Specify the image placeholder for markdown exports.",- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "md_image_placeholder",- "override_skip": false,- "placeholder": "",- "required": false,- "show": true,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": "<!-- image -->"- },- "md_page_break_placeholder": {- "_input_type": "StrInput",- "advanced": true,- "display_name": "Page break placeholder",- "dynamic": false,- "info": "Add this placeholder betweek pages in the markdown output.",- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "md_page_break_placeholder",- "override_skip": false,- "placeholder": "",- "required": false,- "show": true,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": ""- }- },- "tool_mode": false- },- "selected_output": "dataframe",- "showNode": false,- "type": "ExportDoclingDocument"- },- "dragging": false,- "id": "ExportDoclingDocument-zZdRg",- "measured": {- "height": 52,- "width": 192- },- "position": {- "x": 259.70537784351194,- "y": 1766.9263423792859- },- "selected": false,- "type": "genericNode"- },- {- "data": {- "description": "Perform various operations on a Table.",- "display_name": "Table Operations",- "id": "DataFrameOperations-1BWXB",- "node": {- "base_classes": [- "Table"- ],- "beta": false,- "conditional_paths": [],- "custom_fields": {},- "description": "Perform various operations on a Table.",- "display_name": "Table Operations",- "documentation": "https://docs.langflow.org/dataframe-operations",- "edited": false,- "field_order": [- "df",- "operation",- "column_name",- "filter_value",- "filter_operator",- "ascending",- "new_column_name",- "new_column_value",- "columns_to_select",- "num_rows",- "replace_value",- "replacement_value",- "left_dataframe",- "right_dataframe",- "merge_on_column",- "merge_how"- ],- "frozen": false,- "icon": "table",- "last_updated": "2026-04-01T20:50:29.042Z",- "legacy": false,- "metadata": {- "code_hash": "4df73846bdd3",- "dependencies": {- "dependencies": [- {- "name": "pandas",- "version": "2.2.3"- },- {- "name": "lfx",- "version": null- }- ],- "total_dependencies": 2- },- "keywords": [- "dataframe",- "dataframe operations",- "table",- "table operations",- "filter",- "sort",- "merge",- "concatenate",- "drop column",- "rename column",- "add column",- "select columns",- "replace value",- "drop duplicates"- ],- "module": "custom_components.table_operations"- },- "minimized": false,- "output_types": [],- "outputs": [- {- "allows_loop": false,- "cache": true,- "display_name": "Table",- "group_outputs": false,- "loop_types": null,- "method": "perform_operation",- "name": "output",- "options": null,- "required_inputs": null,- "selected": "Table",- "tool_mode": true,- "types": [- "Table"- ],- "value": "__UNDEFINED__"- }- ],- "pinned": false,- "template": {- "_frontend_node_flow_id": {- "value": "5488df7c-b93f-4f87-a446-b67028bc0813"- },- "_frontend_node_folder_id": {- "value": "fe68252b-d9af-4eb8-8307-63e971278546"- },- "_type": "Component",- "ascending": {- "_input_type": "BoolInput",- "advanced": false,- "display_name": "Sort Ascending",- "dynamic": true,- "info": "Whether to sort in ascending order.",- "list": false,- "list_add_label": "Add More",- "name": "ascending",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "bool",- "value": true- },- "code": {- "advanced": true,- "dynamic": true,- "fileTypes": [],- "file_path": "",- "info": "",- "list": false,- "load_from_db": false,- "multiline": true,- "name": "code",- "password": false,- "placeholder": "",- "required": true,- "show": true,- "title_case": false,- "type": "code",- "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"Table Operations\"\n description = \"Perform various operations on a Table.\"\n documentation: str = \"https://docs.langflow.org/dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n metadata = {\n \"keywords\": [\n \"dataframe\",\n \"dataframe operations\",\n \"table\",\n \"table operations\",\n \"filter\",\n \"sort\",\n \"merge\",\n \"concatenate\",\n \"drop column\",\n \"rename column\",\n \"add column\",\n \"select columns\",\n \"replace value\",\n \"drop duplicates\",\n ],\n }\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Concatenate\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Merge\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"Table\",\n info=\"The input DataFrame to operate on. Connect multiple DataFrames for merge or concatenate operations.\",\n required=True,\n is_list=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Concatenate\", \"icon\": \"combine\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Merge\", \"icon\": \"merge\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n DataFrameInput(\n name=\"left_dataframe\",\n display_name=\"Left Table\",\n info=\"The left (primary) DataFrame for merge operations. \"\n \"In a left merge, all rows from this table are preserved.\",\n dynamic=True,\n show=False,\n ),\n DataFrameInput(\n name=\"right_dataframe\",\n display_name=\"Right Table\",\n info=\"The right (secondary) DataFrame for merge operations. \"\n \"In a right merge, all rows from this table are preserved.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"merge_on_column\",\n display_name=\"Merge On Column\",\n info=\"The column name to merge DataFrames on. Must exist in both DataFrames.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"merge_how\",\n display_name=\"Merge Type\",\n options=[\"inner\", \"outer\", \"left\", \"right\"],\n value=\"inner\",\n info=\"Type of merge: inner (intersection), outer (union), left, or right.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"Table\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n \"left_dataframe\",\n \"right_dataframe\",\n \"merge_on_column\",\n \"merge_how\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Merge\":\n build_config[\"left_dataframe\"][\"show\"] = True\n build_config[\"right_dataframe\"][\"show\"] = True\n build_config[\"merge_on_column\"][\"show\"] = True\n build_config[\"merge_how\"][\"show\"] = True\n\n return build_config\n\n def _get_primary_dataframe(self) -> DataFrame:\n \"\"\"Get the first DataFrame from input (handles both single and list inputs).\"\"\"\n if isinstance(self.df, list):\n return self.df[0].copy() if self.df else DataFrame()\n return self.df.copy()\n\n def perform_operation(self) -> DataFrame:\n # Handle SortableListInput format for operation (also supports legacy string format)\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list):\n op = operation_input[0].get(\"name\", \"\") if operation_input else \"\"\n else:\n op = operation_input or \"\"\n\n # Merge and Concatenate use their own inputs, not the primary df\n if op == \"Merge\":\n return self.merge_dataframes()\n if op == \"Concatenate\":\n return self.concatenate_dataframes()\n\n df_copy = self._get_primary_dataframe()\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n\n def concatenate_dataframes(self) -> DataFrame:\n \"\"\"Concatenate multiple DataFrames vertically (stack rows).\"\"\"\n if not isinstance(self.df, list) or len(self.df) == 0:\n return self.df.copy() if self.df is not None else DataFrame()\n\n # If only one DataFrame, return it\n if len(self.df) == 1:\n return self.df[0].copy()\n\n # Concatenate all DataFrames vertically\n concatenated = pd.concat(self.df, ignore_index=True)\n return DataFrame(concatenated)\n\n def merge_dataframes(self) -> DataFrame:\n \"\"\"Merge two DataFrames based on a common column (join operation).\n\n Uses explicit left_dataframe and right_dataframe inputs to give the user\n deterministic control over which DataFrame is primary (left) and which\n is secondary (right) in the merge.\n \"\"\"\n left_df = getattr(self, \"left_dataframe\", None)\n right_df = getattr(self, \"right_dataframe\", None)\n\n if left_df is None:\n return DataFrame()\n\n if right_df is None:\n return left_df.copy()\n\n df_left = left_df.copy()\n df_right = right_df.copy()\n\n merge_on = getattr(self, \"merge_on_column\", None)\n merge_how = getattr(self, \"merge_how\", \"inner\")\n\n if merge_on:\n if merge_on not in df_left.columns:\n msg = f\"Column '{merge_on}' not found in left DataFrame. Available: {list(df_left.columns)}\"\n raise ValueError(msg)\n if merge_on not in df_right.columns:\n msg = f\"Column '{merge_on}' not found in right DataFrame. Available: {list(df_right.columns)}\"\n raise ValueError(msg)\n\n merged = df_left.merge(df_right, on=merge_on, how=merge_how, suffixes=(\"\", \"_right\"))\n else:\n merged = df_left.merge(df_right, left_index=True, right_index=True, how=merge_how, suffixes=(\"\", \"_right\"))\n\n # Combine duplicate columns: use left value if exists, otherwise right value\n cols_to_drop = []\n for col in merged.columns:\n if col.endswith(\"_right\"):\n original_col = col[:-6] # Remove \"_right\" suffix\n if original_col in merged.columns:\n merged[original_col] = merged[original_col].combine_first(merged[col])\n cols_to_drop.append(col)\n\n if cols_to_drop:\n merged = merged.drop(columns=cols_to_drop)\n\n return DataFrame(merged)\n"- },- "column_name": {- "_input_type": "StrInput",- "advanced": false,- "display_name": "Column Name",- "dynamic": true,- "info": "The column name to use for the operation.",- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "column_name",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": ""- },- "columns_to_select": {- "_input_type": "StrInput",- "advanced": false,- "display_name": "Columns to Select",- "dynamic": true,- "info": "",- "list": true,- "list_add_label": "Add More",- "load_from_db": false,- "name": "columns_to_select",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": ""- },- "df": {- "_input_type": "DataFrameInput",- "advanced": false,- "display_name": "Table",- "dynamic": false,- "info": "The input DataFrame to operate on. Connect multiple DataFrames for merge or concatenate operations.",- "input_types": [- "DataFrame",- "Table"- ],- "list": true,- "list_add_label": "Add More",- "name": "df",- "override_skip": false,- "placeholder": "",- "required": true,- "show": true,- "title_case": false,- "tool_mode": false,- "trace_as_input": true,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "other",- "value": ""- },- "filter_operator": {- "_input_type": "DropdownInput",- "advanced": false,- "combobox": false,- "dialog_inputs": {},- "display_name": "Filter Operator",- "dynamic": true,- "external_options": {},- "info": "The operator to apply for filtering rows.",- "name": "filter_operator",- "options": [- "equals",- "not equals",- "contains",- "not contains",- "starts with",- "ends with",- "greater than",- "less than"- ],- "options_metadata": [],- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "toggle": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "str",- "value": "equals"- },- "filter_value": {- "_input_type": "MessageTextInput",- "advanced": false,- "display_name": "Filter Value",- "dynamic": true,- "info": "The value to filter rows by.",- "input_types": [- "Message"- ],- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "filter_value",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_input": true,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": ""- },- "is_refresh": false,- "left_dataframe": {- "_input_type": "DataFrameInput",- "advanced": false,- "display_name": "Left Table",- "dynamic": true,- "info": "The left (primary) DataFrame for merge operations. In a left merge, all rows from this table are preserved.",- "input_types": [- "DataFrame",- "Table"- ],- "list": false,- "list_add_label": "Add More",- "name": "left_dataframe",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_input": true,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "other",- "value": ""- },- "merge_how": {- "_input_type": "DropdownInput",- "advanced": false,- "combobox": false,- "dialog_inputs": {},- "display_name": "Merge Type",- "dynamic": true,- "external_options": {},- "info": "Type of merge: inner (intersection), outer (union), left, or right.",- "name": "merge_how",- "options": [- "inner",- "outer",- "left",- "right"- ],- "options_metadata": [],- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "toggle": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "str",- "value": "inner"- },- "merge_on_column": {- "_input_type": "StrInput",- "advanced": false,- "display_name": "Merge On Column",- "dynamic": true,- "info": "The column name to merge DataFrames on. Must exist in both DataFrames.",+ "info": "","list": false,- "list_add_label": "Add More","load_from_db": false,- "name": "merge_on_column",- "override_skip": false,+ "multiline": true,+ "name": "code",+ "password": false,"placeholder": "",- "required": false,- "show": false,+ "required": true,+ "show": true,"title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": false,- "type": "str",- "value": ""+ "type": "code",+ "value": "from typing import Any\n\nfrom docling_core.types.doc import ImageRefMode\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ExportDoclingDocumentComponent(Component):\n display_name: str = \"Export DoclingDocument\"\n description: str = \"Export DoclingDocument to markdown, html or other formats.\"\n documentation = \"https://docling-project.github.io/docling/\"\n icon = \"Docling\"\n name = \"ExportDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"JSON or Table\",\n info=\"The data with documents to export.\",\n input_types=[\"Data\", \"JSON\", \"DataFrame\", \"Table\"],\n required=True,\n ),\n DropdownInput(\n name=\"export_format\",\n display_name=\"Export format\",\n options=[\"Markdown\", \"HTML\", \"Plaintext\", \"DocTags\"],\n info=\"Select the export format to convert the input.\",\n value=\"Markdown\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"image_mode\",\n display_name=\"Image export mode\",\n options=[\"placeholder\", \"embedded\"],\n info=(\n \"Specify how images are exported in the output. Placeholder will replace the images with a string, \"\n \"whereas Embedded will include them as base64 encoded images.\"\n ),\n value=\"placeholder\",\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder betweek pages in the markdown output.\",\n value=\"\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Exported data\", name=\"data\", method=\"export_document\"),\n Output(display_name=\"Table\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"export_format\" and field_value == \"Markdown\":\n build_config[\"md_image_placeholder\"][\"show\"] = True\n build_config[\"md_page_break_placeholder\"][\"show\"] = True\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value == \"HTML\":\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value in {\"Plaintext\", \"DocTags\"}:\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = False\n\n return build_config\n\n def export_document(self) -> list[Data]:\n documents, warning = extract_docling_documents(self.data_inputs, self.doc_key)\n if warning:\n self.status = warning\n\n results: list[Data] = []\n try:\n image_mode = ImageRefMode(self.image_mode)\n for doc in documents:\n content = \"\"\n if self.export_format == \"Markdown\":\n content = doc.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n elif self.export_format == \"HTML\":\n content = doc.export_to_html(image_mode=image_mode)\n elif self.export_format == \"Plaintext\":\n content = doc.export_to_text()\n elif self.export_format == \"DocTags\":\n content = doc.export_to_doctags()\n\n # Preserve metadata from the DoclingDocument\n metadata: dict = {\"export_format\": self.export_format}\n if hasattr(doc, \"name\") and doc.name:\n metadata[\"name\"] = doc.name\n if hasattr(doc, \"origin\") and doc.origin is not None:\n if hasattr(doc.origin, \"filename\") and doc.origin.filename:\n metadata[\"filename\"] = doc.origin.filename\n if hasattr(doc.origin, \"binary_hash\") and doc.origin.binary_hash:\n metadata[\"document_id\"] = str(doc.origin.binary_hash)\n if hasattr(doc.origin, \"mimetype\") and doc.origin.mimetype:\n metadata[\"mimetype\"] = doc.origin.mimetype\n\n results.append(Data(text=content, data={\"text\": content, **metadata}))\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return results\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.export_document())\n"},- "new_column_name": {- "_input_type": "StrInput",+ "data_inputs": {+ "_input_type": "HandleInput","advanced": false,- "display_name": "New Column Name",- "dynamic": true,- "info": "The new column name when renaming or adding a column.",+ "display_name": "JSON or Table",+ "dynamic": false,+ "info": "The data with documents to export.",+ "input_types": [+ "Data",+ "JSON",+ "DataFrame",+ "Table"+ ],"list": false,"list_add_label": "Add More",- "load_from_db": false,- "name": "new_column_name",+ "name": "data_inputs","override_skip": false,"placeholder": "",- "required": false,+ "required": true,"show": true,"title_case": false,- "tool_mode": false,"trace_as_metadata": true,"track_in_telemetry": false,- "type": "str",- "value": "filename"+ "type": "other",+ "value": ""},- "new_column_value": {+ "doc_key": {"_input_type": "MessageTextInput",- "advanced": false,- "display_name": "New Column Value",- "dynamic": true,- "info": "The value to populate the new column with.",+ "advanced": true,+ "display_name": "Doc Key",+ "dynamic": false,+ "info": "The key to use for the DoclingDocument column.","input_types": ["Message"],"list": false,"list_add_label": "Add More",- "load_from_db": true,- "name": "new_column_value",+ "load_from_db": false,+ "name": "doc_key","override_skip": false,"placeholder": "","required": false,@@ -1722,197 +1188,123 @@"trace_as_metadata": true,"track_in_telemetry": false,"type": "str",- "value": "FILENAME"- },- "num_rows": {- "_input_type": "IntInput",- "advanced": false,- "display_name": "Number of Rows",- "dynamic": true,- "info": "Number of rows to return (for head/tail).",- "list": false,- "list_add_label": "Add More",- "name": "num_rows",- "override_skip": false,- "placeholder": "",- "required": false,- "show": false,- "title_case": false,- "tool_mode": false,- "trace_as_metadata": true,- "track_in_telemetry": true,- "type": "int",- "value": 5+ "value": "doc"},- "operation": {- "_input_type": "SortableListInput",+ "export_format": {+ "_input_type": "DropdownInput","advanced": false,- "display_name": "Operation",+ "combobox": false,+ "dialog_inputs": {},+ "display_name": "Export format","dynamic": false,- "info": "Select the DataFrame operation to perform.",- "limit": 1,- "load_from_db": false,- "name": "operation",+ "external_options": {},+ "info": "Select the export format to convert the input.",+ "name": "export_format","options": [- {- "icon": "plus",- "name": "Add Column"- },- {- "icon": "combine",- "name": "Concatenate"- },- {- "icon": "minus",- "name": "Drop Column"- },- {- "icon": "filter",- "name": "Filter"- },- {- "icon": "arrow-up",- "name": "Head"- },- {- "icon": "merge",- "name": "Merge"- },- {- "icon": "pencil",- "name": "Rename Column"- },- {- "icon": "replace",- "name": "Replace Value"- },- {- "icon": "columns",- "name": "Select Columns"- },- {- "icon": "arrow-up-down",- "name": "Sort"- },- {- "icon": "arrow-down",- "name": "Tail"- },- {- "icon": "copy-x",- "name": "Drop Duplicates"- }+ "Markdown",+ "HTML",+ "Plaintext",+ "DocTags"],+ "options_metadata": [],"override_skip": false,- "placeholder": "Select Operation",+ "placeholder": "","real_time_refresh": true,"required": false,- "search_category": [],"show": true,"title_case": false,+ "toggle": false,"tool_mode": false,"trace_as_metadata": true,- "track_in_telemetry": false,- "type": "sortableList",- "value": [- {- "chosen": false,- "icon": "plus",- "name": "Add Column",- "selected": false- }- ]+ "track_in_telemetry": true,+ "type": "str",+ "value": "Markdown"},- "replace_value": {- "_input_type": "MessageTextInput",+ "image_mode": {+ "_input_type": "DropdownInput","advanced": false,- "display_name": "Value to Replace",- "dynamic": true,- "info": "The value to replace in the column.",- "input_types": [- "Message"+ "combobox": false,+ "dialog_inputs": {},+ "display_name": "Image export mode",+ "dynamic": false,+ "external_options": {},+ "info": "Specify how images are exported in the output. Placeholder will replace the images with a string, whereas Embedded will include them as base64 encoded images.",+ "name": "image_mode",+ "options": [+ "placeholder",+ "embedded"],- "list": false,- "list_add_label": "Add More",- "load_from_db": false,- "name": "replace_value",+ "options_metadata": [],"override_skip": false,"placeholder": "","required": false,- "show": false,+ "show": true,"title_case": false,+ "toggle": false,"tool_mode": false,- "trace_as_input": true,"trace_as_metadata": true,- "track_in_telemetry": false,+ "track_in_telemetry": true,"type": "str",- "value": ""+ "value": "placeholder"},- "replacement_value": {- "_input_type": "MessageTextInput",- "advanced": false,- "display_name": "Replacement Value",- "dynamic": true,- "info": "The value to replace with.",- "input_types": [- "Message"- ],+ "md_image_placeholder": {+ "_input_type": "StrInput",+ "advanced": true,+ "display_name": "Image placeholder",+ "dynamic": false,+ "info": "Specify the image placeholder for markdown exports.","list": false,"list_add_label": "Add More","load_from_db": false,- "name": "replacement_value",+ "name": "md_image_placeholder","override_skip": false,"placeholder": "","required": false,- "show": false,+ "show": true,"title_case": false,"tool_mode": false,- "trace_as_input": true,"trace_as_metadata": true,"track_in_telemetry": false,"type": "str",- "value": ""+ "value": "<!-- image -->"},- "right_dataframe": {- "_input_type": "DataFrameInput",- "advanced": false,- "display_name": "Right Table",- "dynamic": true,- "info": "The right (secondary) DataFrame for merge operations. In a right merge, all rows from this table are preserved.",- "input_types": [- "DataFrame",- "Table"- ],+ "md_page_break_placeholder": {+ "_input_type": "StrInput",+ "advanced": true,+ "display_name": "Page break placeholder",+ "dynamic": false,+ "info": "Add this placeholder betweek pages in the markdown output.","list": false,"list_add_label": "Add More",- "name": "right_dataframe",+ "load_from_db": false,+ "name": "md_page_break_placeholder","override_skip": false,"placeholder": "","required": false,- "show": false,+ "show": true,"title_case": false,"tool_mode": false,- "trace_as_input": true,"trace_as_metadata": true,"track_in_telemetry": false,- "type": "other",+ "type": "str","value": ""}},"tool_mode": false},+ "selected_output": "dataframe","showNode": false,- "type": "DataFrameOperations"+ "type": "ExportDoclingDocument"},"dragging": false,- "id": "DataFrameOperations-1BWXB",+ "id": "ExportDoclingDocument-zZdRg","measured": {"height": 52,"width": 192},"position": {- "x": 533.2200473013957,- "y": 1769.3212323101711+ "x": 259.70537784351194,+ "y": 1766.9263423792859},"selected": false,"type": "genericNode"@@ -2490,8 +1882,8 @@"width": 192},"position": {- "x": 1071.7700474772182,- "y": 1772.3941727700599+ "x": 782.5951114664371,+ "y": 1767.889170620294},"selected": false,"type": "genericNode"@@ -3069,8 +2461,8 @@"width": 192},"position": {- "x": 795.0110500080052,- "y": 1768.4224714700015+ "x": 503.84176795249084,+ "y": 1767.6542910069415},"selected": false,"type": "genericNode"@@ -4058,7 +3450,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"},"_type": "Component","api_base": {@@ -4359,7 +3751,7 @@"show_progress_bar": "show_progress_bar"}},- "name": "text-embedding-3-small",+ "name": "text-embedding-3-large","provider": "OpenAI"}]@@ -4586,7 +3978,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"},"_type": "Component","api_base": {@@ -4887,7 +4279,7 @@"show_progress_bar": "show_progress_bar"}},- "name": "text-embedding-3-small",+ "name": "text-embedding-3-large","provider": "OpenAI"}]@@ -5032,8 +4424,8 @@"width": 320},"position": {- "x": 1313.5212863870001,- "y": 1858.0159458799487+ "x": 1309.9949735963942,+ "y": 1861.5422586705547},"selected": false,"type": "genericNode"@@ -5114,7 +4506,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"},"_type": "Component","api_base": {@@ -5414,7 +4806,7 @@"show_progress_bar": "show_progress_bar"}},- "name": "text-embedding-3-small",+ "name": "text-embedding-3-large","provider": "OpenAI"}]@@ -5566,17 +4958,17 @@}],"viewport": {- "x": 91.09392360442735,- "y": -726.3579579832333,- "zoom": 0.6220209268560802+ "x": 312.12645935679006,+ "y": -430.7105214313415,+ "zoom": 0.42979347188646455}},"description": "Load your data for chat context with Retrieval Augmented Generation.","endpoint_name": null,"id": "5488df7c-b93f-4f87-a446-b67028bc0813","is_component": false,- "last_tested_version": "1.9.6",- "locked": true,+ "last_tested_version": "0.9.6",+ "locked": false,"name": "OpenSearch Ingestion Flow","tags": ["openai",diff --git a/src/config/settings.py b/src/config/settings.pyindex 5e593fc39..b00b7c15d 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -1242,6 +1242,17 @@ class AppClients:default_headers = {"x-api-key": api_key, "Content-Type": "application/json"}headers = {**default_headers, **passed_headers}+ # HTTP headers must be ASCII; non-ASCII free-text globals (e.g. a+ # filename or owner name like こんにちは.pdf / José) would otherwise+ # raise UnicodeEncodeError in httpx. Percent-encode such values so+ # the request can be sent. ASCII values pass through unchanged.+ from utils.langflow_headers import ascii_safe_header_value++ headers = {+ k: ascii_safe_header_value(v) if isinstance(v, str) else v+ for k, v in headers.items()+ }+# Remove Content-Type if explicitly set to None (for file uploads)if headers.get("Content-Type") is None:headers.pop("Content-Type", None)diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex e61376d5f..799aeae79 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -426,7 +426,6 @@ class LangflowFileService:"X-Langflow-Global-Var-OWNER_NAME": str(owner_name),"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),- "X-Langflow-Global-Var-FILENAME": filename,"X-Langflow-Global-Var-MIMETYPE": mimetype,"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),@@ -607,7 +606,6 @@ class LangflowFileService:"X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps([]),"X-Langflow-Global-Var-ALLOWED_PRINCIPALS": json.dumps([]),"X-Langflow-Global-Var-DOCLING_TASK_ID": "",- "X-Langflow-Global-Var-FILENAME": str(docs_url),"X-Langflow-Global-Var-MIMETYPE": "text/html","X-Langflow-Global-Var-FILESIZE": "0","X-Langflow-Global-Var-DOCLING_SERVE_VERIFY_SSL": str(DOCLING_SERVE_VERIFY_SSL).lower(),diff --git a/src/utils/langflow_headers.py b/src/utils/langflow_headers.pyindex 12c24325d..21ba56383 100644--- a/src/utils/langflow_headers.py+++ b/src/utils/langflow_headers.py@@ -1,13 +1,38 @@"""Utility functions for building Langflow request headers."""-from typing import Dict+from urllib.parse import quote+from utils.container_utils import transform_localhost_url+def ascii_safe_header_value(value) -> str:+ """Return an ASCII-only HTTP header value.++ httpx (and HTTP itself) requires header values to be ASCII-encodable, so a+ non-ASCII filename or owner name (e.g. ``こんにちは.pdf`` or ``José``) placed+ into an ``X-Langflow-Global-Var-*`` header raises ``UnicodeEncodeError``+ before the request is sent. ASCII values (including spaces) pass through+ byte-for-byte; only values containing non-ASCII characters are+ percent-encoded so they can be transmitted.++ Note: in the legacy direct-write ingestion path (no ingest-token service+ wired) the FILENAME header value is stored verbatim as the indexed+ ``filename`` column, so a non-ASCII filename lands there percent-encoded.+ The backend-router path (the default) is unaffected: it sources the+ authoritative filename from the ingest JWT context, not this header.+ """+ s = "" if value is None else str(value)+ try:+ s.encode("ascii")+ return s+ except UnicodeEncodeError:+ return quote(s, safe=" /")++def build_ibm_opensearch_vars(credentials: str,prefix: str = "X-LANGFLOW-GLOBAL-VAR-",-) -> Dict[str, str]:+) -> dict[str, str]:"""Build IBM OpenSearch auth vars from a credential string.Supports both ``'Basic <b64>'`` (extracts username/password + JWT) and@@ -18,6 +43,7 @@ def build_ibm_opensearch_vars(result = {f"{prefix}JWT": credentials}if credentials.startswith("Basic "):from auth.ibm_auth import extract_ibm_credentials+username, password = extract_ibm_credentials(credentials)result[f"{prefix}OPENSEARCH_USERNAME"] = usernameresult[f"{prefix}OPENSEARCH_PASSWORD"] = password@@ -25,7 +51,7 @@ def build_ibm_opensearch_vars(async def add_provider_credentials_to_headers(- headers: Dict[str, str],+ headers: dict[str, str],config,flows_service=None,jwt_token: str = None,@@ -53,24 +79,29 @@ async def add_provider_credentials_to_headers(headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_APIKEY"] = str(config.providers.watsonx.api_key)if config.providers.watsonx.project_id:- headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_PROJECT_ID"] = str(config.providers.watsonx.project_id)+ headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_PROJECT_ID"] = str(+ config.providers.watsonx.project_id+ )# Add Ollama endpoint (with localhost transformation)if config.providers.ollama.endpoint:if flows_service:- ollama_endpoint = await flows_service.resolve_ollama_url(config.providers.ollama.endpoint)+ ollama_endpoint = await flows_service.resolve_ollama_url(+ config.providers.ollama.endpoint+ )else:ollama_endpoint = transform_localhost_url(config.providers.ollama.endpoint)headers["X-LANGFLOW-GLOBAL-VAR-OLLAMA_BASE_URL"] = str(ollama_endpoint)# Inject OpenSearch URL so Langflow flows always use the correct endpointfrom config.settings import LANGFLOW_OPENSEARCH_HOST, LANGFLOW_OPENSEARCH_PORT- headers["X-LANGFLOW-GLOBAL-VAR-OPENSEARCH_URL"] = f"https://{LANGFLOW_OPENSEARCH_HOST}:{LANGFLOW_OPENSEARCH_PORT}"++ headers["X-LANGFLOW-GLOBAL-VAR-OPENSEARCH_URL"] = (+ f"https://{LANGFLOW_OPENSEARCH_HOST}:{LANGFLOW_OPENSEARCH_PORT}"+ )# IBM mode: inject OpenSearch Basic credentials as separate global varsfrom config.settings import IBM_AUTH_ENABLED+if IBM_AUTH_ENABLED and jwt_token:headers.update(build_ibm_opensearch_vars(jwt_token, prefix="X-LANGFLOW-GLOBAL-VAR-"))---diff --git a/tests/integration/core/test_non_langflow_ingestion.py b/tests/integration/core/test_non_langflow_ingestion.pyindex 2df654d36..7f5fd6fc4 100644--- a/tests/integration/core/test_non_langflow_ingestion.py+++ b/tests/integration/core/test_non_langflow_ingestion.py@@ -14,6 +14,7 @@ def _purge_modules():for mod in ["api.router","api.connector_router",+ "config.config_manager", # reset cached OpenRAGConfig so env-var overrides take effect"config.settings","auth_middleware","main",@@ -148,6 +149,12 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):csv_path = tmp_path / "correlation_report.csv"csv_path.write_text("header1,header2\nvalue1,value2")+ # DocumentFileProcessor computes document_id as hash_id(file_path), not+ # from Docling's binary_hash field, so derive the expected id here.+ from utils.hash_utils import hash_id as _hash_id++ expected_document_id = _hash_id(str(csv_path))+files = {"file": (csv_path.name,@@ -182,7 +189,7 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):# Check direct OpenSearch doc count for this document IDopensearch_resp = await clients.opensearch.search(index=get_index_name(),- body={"query": {"term": {"document_id": "sha-csv-integration-123"}}},+ body={"query": {"term": {"document_id": expected_document_id}}},)hits = opensearch_resp.get("hits", {}).get("hits", [])@@ -191,7 +198,7 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):assert len(hits) == 2, f"Expected 2 indexed chunks, but got {len(hits)}"for hit in hits:source = hit.get("_source", {})- assert source.get("document_id") == "sha-csv-integration-123"+ assert source.get("document_id") == expected_document_idassert source.get("filename") == "correlation_report.csv"assert source.get("mimetype") == "text/csv"assert "testlargephrase" in source.get("text", "")diff --git a/tests/integration/core/test_onboarding_sample_docs.py b/tests/integration/core/test_onboarding_sample_docs.pyindex 896e227cf..650f379b2 100644--- a/tests/integration/core/test_onboarding_sample_docs.py+++ b/tests/integration/core/test_onboarding_sample_docs.py@@ -67,6 +67,10 @@ async def _require_langflow_ready() -> None:async def isolated_onboarding_docs_workspace(tmp_path: Path, monkeypatch):if not os.getenv("OPENAI_API_KEY"):pytest.skip("OPENAI_API_KEY is required for onboarding sample-doc ingestion")+ if not os.getenv("ANTHROPIC_API_KEY"):+ pytest.skip(+ "ANTHROPIC_API_KEY is required for onboarding sample-doc ingestion (ibm_anthropic.pdf)"+ )docs_dir = Path(__file__).resolve().parents[3] / "openrag-documents"expected_filenames = sorted(diff --git a/tests/unit/test_ascii_safe_header_value.py b/tests/unit/test_ascii_safe_header_value.pynew file mode 100644index 000000000..2bf2ae746--- /dev/null+++ b/tests/unit/test_ascii_safe_header_value.py@@ -0,0 +1,72 @@+"""Pin the ASCII-safe HTTP header encoding used for Langflow global variables.++A non-ASCII filename or owner name placed into an ``X-Langflow-Global-Var-*``+header raised ``UnicodeEncodeError`` in httpx before the request was sent+(httpx requires ASCII-encodable header values). The helper at+`src/utils/langflow_headers.py::ascii_safe_header_value` percent-encodes only+non-ASCII values; ASCII passes through untouched.+"""++import sys+from pathlib import Path++import httpx+import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++from utils.langflow_headers import ascii_safe_header_value # noqa: E402+++ "value,expected",+ [+ # ASCII passes through byte-for-byte, including spaces and slashes.+ ("report.pdf", "report.pdf"),+ ("my report (final).pdf", "my report (final).pdf"),+ ("a/b/c.txt", "a/b/c.txt"),+ ("", ""),+ # None coerces to empty string.+ (None, ""),+ # Non-string ASCII coerces via str().+ (123, "123"),+ ],+)+def test_ascii_values_pass_through(value, expected):+ assert ascii_safe_header_value(value) == expected+++ "value",+ [+ "こんにちは こんにちは.pdf", # Japanese (the reported crash)+ "José García.docx", # accented owner name+ "файл.pdf", # Cyrillic+ "emoji-📄.pdf", # emoji+ ],+)+def test_non_ascii_values_become_ascii_encodable(value):+ encoded = ascii_safe_header_value(value)+ # The whole point: the result must survive httpx's ASCII encoding.+ encoded.encode("ascii")+ # It is percent-encoded, not silently dropped — the value is non-empty+ # and differs from the raw input.+ assert encoded+ assert encoded != value+++def test_headers_dict_survives_httpx_normalization():+ """Regression: a header dict carrying non-ASCII owner metadata must build+ into httpx.Headers without raising (this is exactly where ingestion crashed).+ X-Langflow-Global-Var-FILENAME was removed; OWNER_NAME/OWNER_EMAIL are the+ remaining headers that can carry user-supplied non-ASCII values."""+ headers = {+ "X-Langflow-Global-Var-OWNER_NAME": ascii_safe_header_value("José García"),+ "X-Langflow-Global-Var-OWNER_EMAIL": ascii_safe_header_value("josé@例え.jp"),+ }+ # Previously raised UnicodeEncodeError here.+ httpx.Headers(headers)diff --git a/tests/unit/test_langflow_ingest_callback.py b/tests/unit/test_langflow_ingest_callback.pyindex fabf09e45..0f6236583 100644--- a/tests/unit/test_langflow_ingest_callback.py+++ b/tests/unit/test_langflow_ingest_callback.py@@ -329,7 +329,8 @@ def test_ingest_flows_resolve_callback_config_from_global_vars(flow_path, compondef test_ingest_flows_wire_callback_global_vars_into_opensearch(flow_path, component_id):flow = json.loads(Path(flow_path).read_text(encoding="utf-8"))nodes = {node.get("id"): node for node in flow["data"]["nodes"]}- edges = flow["data"]["edges"]+ component = nodes[component_id]+ template = component["data"]["node"]["template"]expected = {"openrag_ingest_url": "OPENRAG_INGEST_URL","openrag_ingest_token": "OPENRAG_INGEST_TOKEN",@@ -337,18 +338,14 @@ def test_ingest_flows_wire_callback_global_vars_into_opensearch(flow_path, compo}for field_name, variable_name in expected.items():- source_id = f"TextInput-OpenRAGIngest-{field_name.removeprefix('openrag_ingest_')}"- text_node = nodes[source_id]- input_template = text_node["data"]["node"]["template"]["input_value"]+ input_template = template[field_name]assert input_template["value"] == variable_nameassert input_template["load_from_db"] is True+ assert input_template["show"] is True- assert any(- edge.get("source") == source_id- and edge.get("target") == component_id- and edge.get("data", {}).get("targetHandle", {}).get("fieldName") == field_name- for edge in edges- )+ assert template["openrag_ingest_batch_size"]["value"] == 100+ assert template["openrag_ingest_batch_size"].get("load_from_db") is None+ assert template["openrag_ingest_batch_size"]["show"] is True@pytest.mark.parametrize(