Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: universal file names (#1982)

Edwin Joseefd5f677c6d1
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index fd67e465e..f59f7a69d 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -35,7 +35,7 @@ env:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}
# Single-node test cluster never reaches the multi-node readiness counts
- OPENSEARCH_NODE_COUNT_CHECK: "false"
+ OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false"
jobs:
changes:
diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json
index cd9e9d72e..2456116c4 100644
--- a/flows/ingestion_flow.json
+++ b/flows/ingestion_flow.json
@@ -32,35 +32,6 @@
"target": "ExportDoclingDocument-zZdRg",
"targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œJSONœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
},
- {
- "animated": false,
- "className": "",
- "data": {
- "sourceHandle": {
- "dataType": "ExportDoclingDocument",
- "id": "ExportDoclingDocument-zZdRg",
- "name": "dataframe",
- "output_types": [
- "Table"
- ]
- },
- "targetHandle": {
- "fieldName": "df",
- "id": "DataFrameOperations-1BWXB",
- "inputTypes": [
- "DataFrame",
- "Table"
- ],
- "type": "other"
- }
- },
- "id": "reactflow__edge-ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",
- "selected": false,
- "source": "ExportDoclingDocument-zZdRg",
- "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",
- "target": "DataFrameOperations-1BWXB",
- "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
- },
{
"animated": false,
"className": "",
@@ -93,35 +64,6 @@
"target": "SplitText-QIKhg",
"targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œJSONœ,œDataFrameœ,œTableœ,œMessageœ],œtypeœ:œotherœ}"
},
- {
- "animated": false,
- "className": "",
- "data": {
- "sourceHandle": {
- "dataType": "DataFrameOperations",
- "id": "DataFrameOperations-1BWXB",
- "name": "output",
- "output_types": [
- "Table"
- ]
- },
- "targetHandle": {
- "fieldName": "df",
- "id": "DataFrameOperations-9vMrp",
- "inputTypes": [
- "DataFrame",
- "Table"
- ],
- "type": "other"
- }
- },
- "id": "reactflow__edge-DataFrameOperations-1BWXB{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-9vMrp{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",
- "selected": false,
- "source": "DataFrameOperations-1BWXB",
- "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-1BWXBœ,œnameœ:œoutputœ,œoutput_typesœ:[œTableœ]}",
- "target": "DataFrameOperations-9vMrp",
- "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
- },
{
"animated": false,
"className": "",
@@ -264,6 +206,35 @@
"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",
"target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
"targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "ExportDoclingDocument",
+ "id": "ExportDoclingDocument-zZdRg",
+ "name": "dataframe",
+ "output_types": [
+ "Table"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "df",
+ "id": "DataFrameOperations-9vMrp",
+ "inputTypes": [
+ "DataFrame",
+ "Table"
+ ],
+ "type": "other"
+ }
+ },
+ "id": "xy-edge__ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}-DataFrameOperations-9vMrp{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}",
+ "selected": false,
+ "source": "ExportDoclingDocument-zZdRg",
+ "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",
+ "target": "DataFrameOperations-9vMrp",
+ "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-9vMrpœ,œinputTypesœ:[œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
}
],
"nodes": [
@@ -535,8 +506,8 @@
"width": 320
},
"position": {
- "x": 1358.8096961625195,
- "y": 1779.155571003572
+ "x": 1104.7822073166224,
+ "y": 1766.8724817385216
},
"positionAbsolute": {
"x": 1683.4543896546102,
@@ -1156,562 +1127,57 @@
"dynamic": true,
"fileTypes": [],
"file_path": "",
- "info": "",
- "list": false,
- "load_from_db": false,
- "multiline": true,
- "name": "code",
- "password": false,
- "placeholder": "",
- "required": true,
- "show": true,
- "title_case": false,
- "type": "code",
- "value": "from typing import Any\n\nfrom docling_core.types.doc import ImageRefMode\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ExportDoclingDocumentComponent(Component):\n display_name: str = \"Export DoclingDocument\"\n description: str = \"Export DoclingDocument to markdown, html or other formats.\"\n documentation = \"https://docling-project.github.io/docling/\"\n icon = \"Docling\"\n name = \"ExportDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"JSON or Table\",\n info=\"The data with documents to export.\",\n input_types=[\"Data\", \"JSON\", \"DataFrame\", \"Table\"],\n required=True,\n ),\n DropdownInput(\n name=\"export_format\",\n display_name=\"Export format\",\n options=[\"Markdown\", \"HTML\", \"Plaintext\", \"DocTags\"],\n info=\"Select the export format to convert the input.\",\n value=\"Markdown\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"image_mode\",\n display_name=\"Image export mode\",\n options=[\"placeholder\", \"embedded\"],\n info=(\n \"Specify how images are exported in the output. Placeholder will replace the images with a string, \"\n \"whereas Embedded will include them as base64 encoded images.\"\n ),\n value=\"placeholder\",\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder betweek pages in the markdown output.\",\n value=\"\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Exported data\", name=\"data\", method=\"export_document\"),\n Output(display_name=\"Table\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"export_format\" and field_value == \"Markdown\":\n build_config[\"md_image_placeholder\"][\"show\"] = True\n build_config[\"md_page_break_placeholder\"][\"show\"] = True\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value == \"HTML\":\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value in {\"Plaintext\", \"DocTags\"}:\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = False\n\n return build_config\n\n def export_document(self) -> list[Data]:\n documents, warning = extract_docling_documents(self.data_inputs, self.doc_key)\n if warning:\n self.status = warning\n\n results: list[Data] = []\n try:\n image_mode = ImageRefMode(self.image_mode)\n for doc in documents:\n content = \"\"\n if self.export_format == \"Markdown\":\n content = doc.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n elif self.export_format == \"HTML\":\n content = doc.export_to_html(image_mode=image_mode)\n elif self.export_format == \"Plaintext\":\n content = doc.export_to_text()\n elif self.export_format == \"DocTags\":\n content = doc.export_to_doctags()\n\n # Preserve metadata from the DoclingDocument\n metadata: dict = {\"export_format\": self.export_format}\n if hasattr(doc, \"name\") and doc.name:\n metadata[\"name\"] = doc.name\n if hasattr(doc, \"origin\") and doc.origin is not None:\n if hasattr(doc.origin, \"filename\") and doc.origin.filename:\n metadata[\"filename\"] = doc.origin.filename\n if hasattr(doc.origin, \"binary_hash\") and doc.origin.binary_hash:\n metadata[\"document_id\"] = str(doc.origin.binary_hash)\n if hasattr(doc.origin, \"mimetype\") and doc.origin.mimetype:\n metadata[\"mimetype\"] = doc.origin.mimetype\n\n results.append(Data(text=content, data={\"text\": content, **metadata}))\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return results\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.export_document())\n"
- },
- "data_inputs": {
- "_input_type": "HandleInput",
- "advanced": false,
- "display_name": "JSON or Table",
- "dynamic": false,
- "info": "The data with documents to export.",
- "input_types": [
- "Data",
- "JSON",
- "DataFrame",
- "Table"
- ],
- "list": false,
- "list_add_label": "Add More",
- "name": "data_inputs",
- "override_skip": false,
- "placeholder": "",
- "required": true,
- "show": true,
- "title_case": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "other",
- "value": ""
- },
- "doc_key": {
- "_input_type": "MessageTextInput",
- "advanced": true,
- "display_name": "Doc Key",
- "dynamic": false,
- "info": "The key to use for the DoclingDocument column.",
- "input_types": [
- "Message"
- ],
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "doc_key",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": true,
- "title_case": false,
- "tool_mode": false,
- "trace_as_input": true,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": "doc"
- },
- "export_format": {
- "_input_type": "DropdownInput",
- "advanced": false,
- "combobox": false,
- "dialog_inputs": {},
- "display_name": "Export format",
- "dynamic": false,
- "external_options": {},
- "info": "Select the export format to convert the input.",
- "name": "export_format",
- "options": [
- "Markdown",
- "HTML",
- "Plaintext",
- "DocTags"
- ],
- "options_metadata": [],
- "override_skip": false,
- "placeholder": "",
- "real_time_refresh": true,
- "required": false,
- "show": true,
- "title_case": false,
- "toggle": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "str",
- "value": "Markdown"
- },
- "image_mode": {
- "_input_type": "DropdownInput",
- "advanced": false,
- "combobox": false,
- "dialog_inputs": {},
- "display_name": "Image export mode",
- "dynamic": false,
- "external_options": {},
- "info": "Specify how images are exported in the output. Placeholder will replace the images with a string, whereas Embedded will include them as base64 encoded images.",
- "name": "image_mode",
- "options": [
- "placeholder",
- "embedded"
- ],
- "options_metadata": [],
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": true,
- "title_case": false,
- "toggle": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "str",
- "value": "placeholder"
- },
- "md_image_placeholder": {
- "_input_type": "StrInput",
- "advanced": true,
- "display_name": "Image placeholder",
- "dynamic": false,
- "info": "Specify the image placeholder for markdown exports.",
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "md_image_placeholder",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": true,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": "<!-- image -->"
- },
- "md_page_break_placeholder": {
- "_input_type": "StrInput",
- "advanced": true,
- "display_name": "Page break placeholder",
- "dynamic": false,
- "info": "Add this placeholder betweek pages in the markdown output.",
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "md_page_break_placeholder",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": true,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": ""
- }
- },
- "tool_mode": false
- },
- "selected_output": "dataframe",
- "showNode": false,
- "type": "ExportDoclingDocument"
- },
- "dragging": false,
- "id": "ExportDoclingDocument-zZdRg",
- "measured": {
- "height": 52,
- "width": 192
- },
- "position": {
- "x": 259.70537784351194,
- "y": 1766.9263423792859
- },
- "selected": false,
- "type": "genericNode"
- },
- {
- "data": {
- "description": "Perform various operations on a Table.",
- "display_name": "Table Operations",
- "id": "DataFrameOperations-1BWXB",
- "node": {
- "base_classes": [
- "Table"
- ],
- "beta": false,
- "conditional_paths": [],
- "custom_fields": {},
- "description": "Perform various operations on a Table.",
- "display_name": "Table Operations",
- "documentation": "https://docs.langflow.org/dataframe-operations",
- "edited": false,
- "field_order": [
- "df",
- "operation",
- "column_name",
- "filter_value",
- "filter_operator",
- "ascending",
- "new_column_name",
- "new_column_value",
- "columns_to_select",
- "num_rows",
- "replace_value",
- "replacement_value",
- "left_dataframe",
- "right_dataframe",
- "merge_on_column",
- "merge_how"
- ],
- "frozen": false,
- "icon": "table",
- "last_updated": "2026-04-01T20:50:29.042Z",
- "legacy": false,
- "metadata": {
- "code_hash": "4df73846bdd3",
- "dependencies": {
- "dependencies": [
- {
- "name": "pandas",
- "version": "2.2.3"
- },
- {
- "name": "lfx",
- "version": null
- }
- ],
- "total_dependencies": 2
- },
- "keywords": [
- "dataframe",
- "dataframe operations",
- "table",
- "table operations",
- "filter",
- "sort",
- "merge",
- "concatenate",
- "drop column",
- "rename column",
- "add column",
- "select columns",
- "replace value",
- "drop duplicates"
- ],
- "module": "custom_components.table_operations"
- },
- "minimized": false,
- "output_types": [],
- "outputs": [
- {
- "allows_loop": false,
- "cache": true,
- "display_name": "Table",
- "group_outputs": false,
- "loop_types": null,
- "method": "perform_operation",
- "name": "output",
- "options": null,
- "required_inputs": null,
- "selected": "Table",
- "tool_mode": true,
- "types": [
- "Table"
- ],
- "value": "__UNDEFINED__"
- }
- ],
- "pinned": false,
- "template": {
- "_frontend_node_flow_id": {
- "value": "5488df7c-b93f-4f87-a446-b67028bc0813"
- },
- "_frontend_node_folder_id": {
- "value": "fe68252b-d9af-4eb8-8307-63e971278546"
- },
- "_type": "Component",
- "ascending": {
- "_input_type": "BoolInput",
- "advanced": false,
- "display_name": "Sort Ascending",
- "dynamic": true,
- "info": "Whether to sort in ascending order.",
- "list": false,
- "list_add_label": "Add More",
- "name": "ascending",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "bool",
- "value": true
- },
- "code": {
- "advanced": true,
- "dynamic": true,
- "fileTypes": [],
- "file_path": "",
- "info": "",
- "list": false,
- "load_from_db": false,
- "multiline": true,
- "name": "code",
- "password": false,
- "placeholder": "",
- "required": true,
- "show": true,
- "title_case": false,
- "type": "code",
- "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"Table Operations\"\n description = \"Perform various operations on a Table.\"\n documentation: str = \"https://docs.langflow.org/dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n metadata = {\n \"keywords\": [\n \"dataframe\",\n \"dataframe operations\",\n \"table\",\n \"table operations\",\n \"filter\",\n \"sort\",\n \"merge\",\n \"concatenate\",\n \"drop column\",\n \"rename column\",\n \"add column\",\n \"select columns\",\n \"replace value\",\n \"drop duplicates\",\n ],\n }\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Concatenate\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Merge\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"Table\",\n info=\"The input DataFrame to operate on. Connect multiple DataFrames for merge or concatenate operations.\",\n required=True,\n is_list=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Concatenate\", \"icon\": \"combine\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Merge\", \"icon\": \"merge\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n DataFrameInput(\n name=\"left_dataframe\",\n display_name=\"Left Table\",\n info=\"The left (primary) DataFrame for merge operations. \"\n \"In a left merge, all rows from this table are preserved.\",\n dynamic=True,\n show=False,\n ),\n DataFrameInput(\n name=\"right_dataframe\",\n display_name=\"Right Table\",\n info=\"The right (secondary) DataFrame for merge operations. \"\n \"In a right merge, all rows from this table are preserved.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"merge_on_column\",\n display_name=\"Merge On Column\",\n info=\"The column name to merge DataFrames on. Must exist in both DataFrames.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"merge_how\",\n display_name=\"Merge Type\",\n options=[\"inner\", \"outer\", \"left\", \"right\"],\n value=\"inner\",\n info=\"Type of merge: inner (intersection), outer (union), left, or right.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"Table\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n \"left_dataframe\",\n \"right_dataframe\",\n \"merge_on_column\",\n \"merge_how\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Merge\":\n build_config[\"left_dataframe\"][\"show\"] = True\n build_config[\"right_dataframe\"][\"show\"] = True\n build_config[\"merge_on_column\"][\"show\"] = True\n build_config[\"merge_how\"][\"show\"] = True\n\n return build_config\n\n def _get_primary_dataframe(self) -> DataFrame:\n \"\"\"Get the first DataFrame from input (handles both single and list inputs).\"\"\"\n if isinstance(self.df, list):\n return self.df[0].copy() if self.df else DataFrame()\n return self.df.copy()\n\n def perform_operation(self) -> DataFrame:\n # Handle SortableListInput format for operation (also supports legacy string format)\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list):\n op = operation_input[0].get(\"name\", \"\") if operation_input else \"\"\n else:\n op = operation_input or \"\"\n\n # Merge and Concatenate use their own inputs, not the primary df\n if op == \"Merge\":\n return self.merge_dataframes()\n if op == \"Concatenate\":\n return self.concatenate_dataframes()\n\n df_copy = self._get_primary_dataframe()\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n\n def concatenate_dataframes(self) -> DataFrame:\n \"\"\"Concatenate multiple DataFrames vertically (stack rows).\"\"\"\n if not isinstance(self.df, list) or len(self.df) == 0:\n return self.df.copy() if self.df is not None else DataFrame()\n\n # If only one DataFrame, return it\n if len(self.df) == 1:\n return self.df[0].copy()\n\n # Concatenate all DataFrames vertically\n concatenated = pd.concat(self.df, ignore_index=True)\n return DataFrame(concatenated)\n\n def merge_dataframes(self) -> DataFrame:\n \"\"\"Merge two DataFrames based on a common column (join operation).\n\n Uses explicit left_dataframe and right_dataframe inputs to give the user\n deterministic control over which DataFrame is primary (left) and which\n is secondary (right) in the merge.\n \"\"\"\n left_df = getattr(self, \"left_dataframe\", None)\n right_df = getattr(self, \"right_dataframe\", None)\n\n if left_df is None:\n return DataFrame()\n\n if right_df is None:\n return left_df.copy()\n\n df_left = left_df.copy()\n df_right = right_df.copy()\n\n merge_on = getattr(self, \"merge_on_column\", None)\n merge_how = getattr(self, \"merge_how\", \"inner\")\n\n if merge_on:\n if merge_on not in df_left.columns:\n msg = f\"Column '{merge_on}' not found in left DataFrame. Available: {list(df_left.columns)}\"\n raise ValueError(msg)\n if merge_on not in df_right.columns:\n msg = f\"Column '{merge_on}' not found in right DataFrame. Available: {list(df_right.columns)}\"\n raise ValueError(msg)\n\n merged = df_left.merge(df_right, on=merge_on, how=merge_how, suffixes=(\"\", \"_right\"))\n else:\n merged = df_left.merge(df_right, left_index=True, right_index=True, how=merge_how, suffixes=(\"\", \"_right\"))\n\n # Combine duplicate columns: use left value if exists, otherwise right value\n cols_to_drop = []\n for col in merged.columns:\n if col.endswith(\"_right\"):\n original_col = col[:-6] # Remove \"_right\" suffix\n if original_col in merged.columns:\n merged[original_col] = merged[original_col].combine_first(merged[col])\n cols_to_drop.append(col)\n\n if cols_to_drop:\n merged = merged.drop(columns=cols_to_drop)\n\n return DataFrame(merged)\n"
- },
- "column_name": {
- "_input_type": "StrInput",
- "advanced": false,
- "display_name": "Column Name",
- "dynamic": true,
- "info": "The column name to use for the operation.",
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "column_name",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": ""
- },
- "columns_to_select": {
- "_input_type": "StrInput",
- "advanced": false,
- "display_name": "Columns to Select",
- "dynamic": true,
- "info": "",
- "list": true,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "columns_to_select",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": ""
- },
- "df": {
- "_input_type": "DataFrameInput",
- "advanced": false,
- "display_name": "Table",
- "dynamic": false,
- "info": "The input DataFrame to operate on. Connect multiple DataFrames for merge or concatenate operations.",
- "input_types": [
- "DataFrame",
- "Table"
- ],
- "list": true,
- "list_add_label": "Add More",
- "name": "df",
- "override_skip": false,
- "placeholder": "",
- "required": true,
- "show": true,
- "title_case": false,
- "tool_mode": false,
- "trace_as_input": true,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "other",
- "value": ""
- },
- "filter_operator": {
- "_input_type": "DropdownInput",
- "advanced": false,
- "combobox": false,
- "dialog_inputs": {},
- "display_name": "Filter Operator",
- "dynamic": true,
- "external_options": {},
- "info": "The operator to apply for filtering rows.",
- "name": "filter_operator",
- "options": [
- "equals",
- "not equals",
- "contains",
- "not contains",
- "starts with",
- "ends with",
- "greater than",
- "less than"
- ],
- "options_metadata": [],
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "toggle": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "str",
- "value": "equals"
- },
- "filter_value": {
- "_input_type": "MessageTextInput",
- "advanced": false,
- "display_name": "Filter Value",
- "dynamic": true,
- "info": "The value to filter rows by.",
- "input_types": [
- "Message"
- ],
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "filter_value",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_input": true,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": ""
- },
- "is_refresh": false,
- "left_dataframe": {
- "_input_type": "DataFrameInput",
- "advanced": false,
- "display_name": "Left Table",
- "dynamic": true,
- "info": "The left (primary) DataFrame for merge operations. In a left merge, all rows from this table are preserved.",
- "input_types": [
- "DataFrame",
- "Table"
- ],
- "list": false,
- "list_add_label": "Add More",
- "name": "left_dataframe",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_input": true,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "other",
- "value": ""
- },
- "merge_how": {
- "_input_type": "DropdownInput",
- "advanced": false,
- "combobox": false,
- "dialog_inputs": {},
- "display_name": "Merge Type",
- "dynamic": true,
- "external_options": {},
- "info": "Type of merge: inner (intersection), outer (union), left, or right.",
- "name": "merge_how",
- "options": [
- "inner",
- "outer",
- "left",
- "right"
- ],
- "options_metadata": [],
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "toggle": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "str",
- "value": "inner"
- },
- "merge_on_column": {
- "_input_type": "StrInput",
- "advanced": false,
- "display_name": "Merge On Column",
- "dynamic": true,
- "info": "The column name to merge DataFrames on. Must exist in both DataFrames.",
+ "info": "",
"list": false,
- "list_add_label": "Add More",
"load_from_db": false,
- "name": "merge_on_column",
- "override_skip": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
"placeholder": "",
- "required": false,
- "show": false,
+ "required": true,
+ "show": true,
"title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "str",
- "value": ""
+ "type": "code",
+ "value": "from typing import Any\n\nfrom docling_core.types.doc import ImageRefMode\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ExportDoclingDocumentComponent(Component):\n display_name: str = \"Export DoclingDocument\"\n description: str = \"Export DoclingDocument to markdown, html or other formats.\"\n documentation = \"https://docling-project.github.io/docling/\"\n icon = \"Docling\"\n name = \"ExportDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"JSON or Table\",\n info=\"The data with documents to export.\",\n input_types=[\"Data\", \"JSON\", \"DataFrame\", \"Table\"],\n required=True,\n ),\n DropdownInput(\n name=\"export_format\",\n display_name=\"Export format\",\n options=[\"Markdown\", \"HTML\", \"Plaintext\", \"DocTags\"],\n info=\"Select the export format to convert the input.\",\n value=\"Markdown\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"image_mode\",\n display_name=\"Image export mode\",\n options=[\"placeholder\", \"embedded\"],\n info=(\n \"Specify how images are exported in the output. Placeholder will replace the images with a string, \"\n \"whereas Embedded will include them as base64 encoded images.\"\n ),\n value=\"placeholder\",\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"<!-- image -->\",\n advanced=True,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder betweek pages in the markdown output.\",\n value=\"\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Exported data\", name=\"data\", method=\"export_document\"),\n Output(display_name=\"Table\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"export_format\" and field_value == \"Markdown\":\n build_config[\"md_image_placeholder\"][\"show\"] = True\n build_config[\"md_page_break_placeholder\"][\"show\"] = True\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value == \"HTML\":\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value in {\"Plaintext\", \"DocTags\"}:\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = False\n\n return build_config\n\n def export_document(self) -> list[Data]:\n documents, warning = extract_docling_documents(self.data_inputs, self.doc_key)\n if warning:\n self.status = warning\n\n results: list[Data] = []\n try:\n image_mode = ImageRefMode(self.image_mode)\n for doc in documents:\n content = \"\"\n if self.export_format == \"Markdown\":\n content = doc.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n elif self.export_format == \"HTML\":\n content = doc.export_to_html(image_mode=image_mode)\n elif self.export_format == \"Plaintext\":\n content = doc.export_to_text()\n elif self.export_format == \"DocTags\":\n content = doc.export_to_doctags()\n\n # Preserve metadata from the DoclingDocument\n metadata: dict = {\"export_format\": self.export_format}\n if hasattr(doc, \"name\") and doc.name:\n metadata[\"name\"] = doc.name\n if hasattr(doc, \"origin\") and doc.origin is not None:\n if hasattr(doc.origin, \"filename\") and doc.origin.filename:\n metadata[\"filename\"] = doc.origin.filename\n if hasattr(doc.origin, \"binary_hash\") and doc.origin.binary_hash:\n metadata[\"document_id\"] = str(doc.origin.binary_hash)\n if hasattr(doc.origin, \"mimetype\") and doc.origin.mimetype:\n metadata[\"mimetype\"] = doc.origin.mimetype\n\n results.append(Data(text=content, data={\"text\": content, **metadata}))\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return results\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.export_document())\n"
},
- "new_column_name": {
- "_input_type": "StrInput",
+ "data_inputs": {
+ "_input_type": "HandleInput",
"advanced": false,
- "display_name": "New Column Name",
- "dynamic": true,
- "info": "The new column name when renaming or adding a column.",
+ "display_name": "JSON or Table",
+ "dynamic": false,
+ "info": "The data with documents to export.",
+ "input_types": [
+ "Data",
+ "JSON",
+ "DataFrame",
+ "Table"
+ ],
"list": false,
"list_add_label": "Add More",
- "load_from_db": false,
- "name": "new_column_name",
+ "name": "data_inputs",
"override_skip": false,
"placeholder": "",
- "required": false,
+ "required": true,
"show": true,
"title_case": false,
- "tool_mode": false,
"trace_as_metadata": true,
"track_in_telemetry": false,
- "type": "str",
- "value": "filename"
+ "type": "other",
+ "value": ""
},
- "new_column_value": {
+ "doc_key": {
"_input_type": "MessageTextInput",
- "advanced": false,
- "display_name": "New Column Value",
- "dynamic": true,
- "info": "The value to populate the new column with.",
+ "advanced": true,
+ "display_name": "Doc Key",
+ "dynamic": false,
+ "info": "The key to use for the DoclingDocument column.",
"input_types": [
"Message"
],
"list": false,
"list_add_label": "Add More",
- "load_from_db": true,
- "name": "new_column_value",
+ "load_from_db": false,
+ "name": "doc_key",
"override_skip": false,
"placeholder": "",
"required": false,
@@ -1722,197 +1188,123 @@
"trace_as_metadata": true,
"track_in_telemetry": false,
"type": "str",
- "value": "FILENAME"
- },
- "num_rows": {
- "_input_type": "IntInput",
- "advanced": false,
- "display_name": "Number of Rows",
- "dynamic": true,
- "info": "Number of rows to return (for head/tail).",
- "list": false,
- "list_add_label": "Add More",
- "name": "num_rows",
- "override_skip": false,
- "placeholder": "",
- "required": false,
- "show": false,
- "title_case": false,
- "tool_mode": false,
- "trace_as_metadata": true,
- "track_in_telemetry": true,
- "type": "int",
- "value": 5
+ "value": "doc"
},
- "operation": {
- "_input_type": "SortableListInput",
+ "export_format": {
+ "_input_type": "DropdownInput",
"advanced": false,
- "display_name": "Operation",
+ "combobox": false,
+ "dialog_inputs": {},
+ "display_name": "Export format",
"dynamic": false,
- "info": "Select the DataFrame operation to perform.",
- "limit": 1,
- "load_from_db": false,
- "name": "operation",
+ "external_options": {},
+ "info": "Select the export format to convert the input.",
+ "name": "export_format",
"options": [
- {
- "icon": "plus",
- "name": "Add Column"
- },
- {
- "icon": "combine",
- "name": "Concatenate"
- },
- {
- "icon": "minus",
- "name": "Drop Column"
- },
- {
- "icon": "filter",
- "name": "Filter"
- },
- {
- "icon": "arrow-up",
- "name": "Head"
- },
- {
- "icon": "merge",
- "name": "Merge"
- },
- {
- "icon": "pencil",
- "name": "Rename Column"
- },
- {
- "icon": "replace",
- "name": "Replace Value"
- },
- {
- "icon": "columns",
- "name": "Select Columns"
- },
- {
- "icon": "arrow-up-down",
- "name": "Sort"
- },
- {
- "icon": "arrow-down",
- "name": "Tail"
- },
- {
- "icon": "copy-x",
- "name": "Drop Duplicates"
- }
+ "Markdown",
+ "HTML",
+ "Plaintext",
+ "DocTags"
],
+ "options_metadata": [],
"override_skip": false,
- "placeholder": "Select Operation",
+ "placeholder": "",
"real_time_refresh": true,
"required": false,
- "search_category": [],
"show": true,
"title_case": false,
+ "toggle": false,
"tool_mode": false,
"trace_as_metadata": true,
- "track_in_telemetry": false,
- "type": "sortableList",
- "value": [
- {
- "chosen": false,
- "icon": "plus",
- "name": "Add Column",
- "selected": false
- }
- ]
+ "track_in_telemetry": true,
+ "type": "str",
+ "value": "Markdown"
},
- "replace_value": {
- "_input_type": "MessageTextInput",
+ "image_mode": {
+ "_input_type": "DropdownInput",
"advanced": false,
- "display_name": "Value to Replace",
- "dynamic": true,
- "info": "The value to replace in the column.",
- "input_types": [
- "Message"
+ "combobox": false,
+ "dialog_inputs": {},
+ "display_name": "Image export mode",
+ "dynamic": false,
+ "external_options": {},
+ "info": "Specify how images are exported in the output. Placeholder will replace the images with a string, whereas Embedded will include them as base64 encoded images.",
+ "name": "image_mode",
+ "options": [
+ "placeholder",
+ "embedded"
],
- "list": false,
- "list_add_label": "Add More",
- "load_from_db": false,
- "name": "replace_value",
+ "options_metadata": [],
"override_skip": false,
"placeholder": "",
"required": false,
- "show": false,
+ "show": true,
"title_case": false,
+ "toggle": false,
"tool_mode": false,
- "trace_as_input": true,
"trace_as_metadata": true,
- "track_in_telemetry": false,
+ "track_in_telemetry": true,
"type": "str",
- "value": ""
+ "value": "placeholder"
},
- "replacement_value": {
- "_input_type": "MessageTextInput",
- "advanced": false,
- "display_name": "Replacement Value",
- "dynamic": true,
- "info": "The value to replace with.",
- "input_types": [
- "Message"
- ],
+ "md_image_placeholder": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "Image placeholder",
+ "dynamic": false,
+ "info": "Specify the image placeholder for markdown exports.",
"list": false,
"list_add_label": "Add More",
"load_from_db": false,
- "name": "replacement_value",
+ "name": "md_image_placeholder",
"override_skip": false,
"placeholder": "",
"required": false,
- "show": false,
+ "show": true,
"title_case": false,
"tool_mode": false,
- "trace_as_input": true,
"trace_as_metadata": true,
"track_in_telemetry": false,
"type": "str",
- "value": ""
+ "value": "<!-- image -->"
},
- "right_dataframe": {
- "_input_type": "DataFrameInput",
- "advanced": false,
- "display_name": "Right Table",
- "dynamic": true,
- "info": "The right (secondary) DataFrame for merge operations. In a right merge, all rows from this table are preserved.",
- "input_types": [
- "DataFrame",
- "Table"
- ],
+ "md_page_break_placeholder": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "Page break placeholder",
+ "dynamic": false,
+ "info": "Add this placeholder betweek pages in the markdown output.",
"list": false,
"list_add_label": "Add More",
- "name": "right_dataframe",
+ "load_from_db": false,
+ "name": "md_page_break_placeholder",
"override_skip": false,
"placeholder": "",
"required": false,
- "show": false,
+ "show": true,
"title_case": false,
"tool_mode": false,
- "trace_as_input": true,
"trace_as_metadata": true,
"track_in_telemetry": false,
- "type": "other",
+ "type": "str",
"value": ""
}
},
"tool_mode": false
},
+ "selected_output": "dataframe",
"showNode": false,
- "type": "DataFrameOperations"
+ "type": "ExportDoclingDocument"
},
"dragging": false,
- "id": "DataFrameOperations-1BWXB",
+ "id": "ExportDoclingDocument-zZdRg",
"measured": {
"height": 52,
"width": 192
},
"position": {
- "x": 533.2200473013957,
- "y": 1769.3212323101711
+ "x": 259.70537784351194,
+ "y": 1766.9263423792859
},
"selected": false,
"type": "genericNode"
@@ -2490,8 +1882,8 @@
"width": 192
},
"position": {
- "x": 1071.7700474772182,
- "y": 1772.3941727700599
+ "x": 782.5951114664371,
+ "y": 1767.889170620294
},
"selected": false,
"type": "genericNode"
@@ -3069,8 +2461,8 @@
"width": 192
},
"position": {
- "x": 795.0110500080052,
- "y": 1768.4224714700015
+ "x": 503.84176795249084,
+ "y": 1767.6542910069415
},
"selected": false,
"type": "genericNode"
@@ -4058,7 +3450,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"
+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"
},
"_type": "Component",
"api_base": {
@@ -4359,7 +3751,7 @@
"show_progress_bar": "show_progress_bar"
}
},
- "name": "text-embedding-3-small",
+ "name": "text-embedding-3-large",
"provider": "OpenAI"
}
]
@@ -4586,7 +3978,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"
+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"
},
"_type": "Component",
"api_base": {
@@ -4887,7 +4279,7 @@
"show_progress_bar": "show_progress_bar"
}
},
- "name": "text-embedding-3-small",
+ "name": "text-embedding-3-large",
"provider": "OpenAI"
}
]
@@ -5032,8 +4424,8 @@
"width": 320
},
"position": {
- "x": 1313.5212863870001,
- "y": 1858.0159458799487
+ "x": 1309.9949735963942,
+ "y": 1861.5422586705547
},
"selected": false,
"type": "genericNode"
@@ -5114,7 +4506,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "dcac372c-a803-4877-8e1a-83e9adcd2972"
+ "value": "203c3612-3397-4ae9-8ef8-dbacfad40a39"
},
"_type": "Component",
"api_base": {
@@ -5414,7 +4806,7 @@
"show_progress_bar": "show_progress_bar"
}
},
- "name": "text-embedding-3-small",
+ "name": "text-embedding-3-large",
"provider": "OpenAI"
}
]
@@ -5566,17 +4958,17 @@
}
],
"viewport": {
- "x": 91.09392360442735,
- "y": -726.3579579832333,
- "zoom": 0.6220209268560802
+ "x": 312.12645935679006,
+ "y": -430.7105214313415,
+ "zoom": 0.42979347188646455
}
},
"description": "Load your data for chat context with Retrieval Augmented Generation.",
"endpoint_name": null,
"id": "5488df7c-b93f-4f87-a446-b67028bc0813",
"is_component": false,
- "last_tested_version": "1.9.6",
- "locked": true,
+ "last_tested_version": "0.9.6",
+ "locked": false,
"name": "OpenSearch Ingestion Flow",
"tags": [
"openai",
diff --git a/src/config/settings.py b/src/config/settings.py
index 5e593fc39..b00b7c15d 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -1242,6 +1242,17 @@ class AppClients:
default_headers = {"x-api-key": api_key, "Content-Type": "application/json"}
headers = {**default_headers, **passed_headers}
+ # HTTP headers must be ASCII; non-ASCII free-text globals (e.g. a
+ # filename or owner name like こんにちは.pdf / José) would otherwise
+ # raise UnicodeEncodeError in httpx. Percent-encode such values so
+ # the request can be sent. ASCII values pass through unchanged.
+ from utils.langflow_headers import ascii_safe_header_value
+
+ headers = {
+ k: ascii_safe_header_value(v) if isinstance(v, str) else v
+ for k, v in headers.items()
+ }
+
# Remove Content-Type if explicitly set to None (for file uploads)
if headers.get("Content-Type") is None:
headers.pop("Content-Type", None)
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index e61376d5f..799aeae79 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -426,7 +426,6 @@ class LangflowFileService:
"X-Langflow-Global-Var-OWNER_NAME": str(owner_name),
"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
- "X-Langflow-Global-Var-FILENAME": filename,
"X-Langflow-Global-Var-MIMETYPE": mimetype,
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
@@ -607,7 +606,6 @@ class LangflowFileService:
"X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps([]),
"X-Langflow-Global-Var-ALLOWED_PRINCIPALS": json.dumps([]),
"X-Langflow-Global-Var-DOCLING_TASK_ID": "",
- "X-Langflow-Global-Var-FILENAME": str(docs_url),
"X-Langflow-Global-Var-MIMETYPE": "text/html",
"X-Langflow-Global-Var-FILESIZE": "0",
"X-Langflow-Global-Var-DOCLING_SERVE_VERIFY_SSL": str(DOCLING_SERVE_VERIFY_SSL).lower(),
diff --git a/src/utils/langflow_headers.py b/src/utils/langflow_headers.py
index 12c24325d..21ba56383 100644
--- a/src/utils/langflow_headers.py
+++ b/src/utils/langflow_headers.py
@@ -1,13 +1,38 @@
"""Utility functions for building Langflow request headers."""
-from typing import Dict
+from urllib.parse import quote
+
from utils.container_utils import transform_localhost_url
+def ascii_safe_header_value(value) -> str:
+ """Return an ASCII-only HTTP header value.
+
+ httpx (and HTTP itself) requires header values to be ASCII-encodable, so a
+ non-ASCII filename or owner name (e.g. ``こんにちは.pdf`` or ``José``) placed
+ into an ``X-Langflow-Global-Var-*`` header raises ``UnicodeEncodeError``
+ before the request is sent. ASCII values (including spaces) pass through
+ byte-for-byte; only values containing non-ASCII characters are
+ percent-encoded so they can be transmitted.
+
+ Note: in the legacy direct-write ingestion path (no ingest-token service
+ wired) the FILENAME header value is stored verbatim as the indexed
+ ``filename`` column, so a non-ASCII filename lands there percent-encoded.
+ The backend-router path (the default) is unaffected: it sources the
+ authoritative filename from the ingest JWT context, not this header.
+ """
+ s = "" if value is None else str(value)
+ try:
+ s.encode("ascii")
+ return s
+ except UnicodeEncodeError:
+ return quote(s, safe=" /")
+
+
def build_ibm_opensearch_vars(
credentials: str,
prefix: str = "X-LANGFLOW-GLOBAL-VAR-",
-) -> Dict[str, str]:
+) -> dict[str, str]:
"""Build IBM OpenSearch auth vars from a credential string.
Supports both ``'Basic <b64>'`` (extracts username/password + JWT) and
@@ -18,6 +43,7 @@ def build_ibm_opensearch_vars(
result = {f"{prefix}JWT": credentials}
if credentials.startswith("Basic "):
from auth.ibm_auth import extract_ibm_credentials
+
username, password = extract_ibm_credentials(credentials)
result[f"{prefix}OPENSEARCH_USERNAME"] = username
result[f"{prefix}OPENSEARCH_PASSWORD"] = password
@@ -25,7 +51,7 @@ def build_ibm_opensearch_vars(
async def add_provider_credentials_to_headers(
- headers: Dict[str, str],
+ headers: dict[str, str],
config,
flows_service=None,
jwt_token: str = None,
@@ -53,24 +79,29 @@ async def add_provider_credentials_to_headers(
headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_APIKEY"] = str(config.providers.watsonx.api_key)
if config.providers.watsonx.project_id:
- headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_PROJECT_ID"] = str(config.providers.watsonx.project_id)
+ headers["X-LANGFLOW-GLOBAL-VAR-WATSONX_PROJECT_ID"] = str(
+ config.providers.watsonx.project_id
+ )
# Add Ollama endpoint (with localhost transformation)
if config.providers.ollama.endpoint:
if flows_service:
- ollama_endpoint = await flows_service.resolve_ollama_url(config.providers.ollama.endpoint)
+ ollama_endpoint = await flows_service.resolve_ollama_url(
+ config.providers.ollama.endpoint
+ )
else:
ollama_endpoint = transform_localhost_url(config.providers.ollama.endpoint)
headers["X-LANGFLOW-GLOBAL-VAR-OLLAMA_BASE_URL"] = str(ollama_endpoint)
# Inject OpenSearch URL so Langflow flows always use the correct endpoint
from config.settings import LANGFLOW_OPENSEARCH_HOST, LANGFLOW_OPENSEARCH_PORT
- headers["X-LANGFLOW-GLOBAL-VAR-OPENSEARCH_URL"] = f"https://{LANGFLOW_OPENSEARCH_HOST}:{LANGFLOW_OPENSEARCH_PORT}"
+
+ headers["X-LANGFLOW-GLOBAL-VAR-OPENSEARCH_URL"] = (
+ f"https://{LANGFLOW_OPENSEARCH_HOST}:{LANGFLOW_OPENSEARCH_PORT}"
+ )
# IBM mode: inject OpenSearch Basic credentials as separate global vars
from config.settings import IBM_AUTH_ENABLED
+
if IBM_AUTH_ENABLED and jwt_token:
headers.update(build_ibm_opensearch_vars(jwt_token, prefix="X-LANGFLOW-GLOBAL-VAR-"))
-
-
-
diff --git a/tests/integration/core/test_non_langflow_ingestion.py b/tests/integration/core/test_non_langflow_ingestion.py
index 2df654d36..7f5fd6fc4 100644
--- a/tests/integration/core/test_non_langflow_ingestion.py
+++ b/tests/integration/core/test_non_langflow_ingestion.py
@@ -14,6 +14,7 @@ def _purge_modules():
for mod in [
"api.router",
"api.connector_router",
+ "config.config_manager", # reset cached OpenRAGConfig so env-var overrides take effect
"config.settings",
"auth_middleware",
"main",
@@ -148,6 +149,12 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
csv_path = tmp_path / "correlation_report.csv"
csv_path.write_text("header1,header2\nvalue1,value2")
+ # DocumentFileProcessor computes document_id as hash_id(file_path), not
+ # from Docling's binary_hash field, so derive the expected id here.
+ from utils.hash_utils import hash_id as _hash_id
+
+ expected_document_id = _hash_id(str(csv_path))
+
files = {
"file": (
csv_path.name,
@@ -182,7 +189,7 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
# Check direct OpenSearch doc count for this document ID
opensearch_resp = await clients.opensearch.search(
index=get_index_name(),
- body={"query": {"term": {"document_id": "sha-csv-integration-123"}}},
+ body={"query": {"term": {"document_id": expected_document_id}}},
)
hits = opensearch_resp.get("hits", {}).get("hits", [])
@@ -191,7 +198,7 @@ async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
assert len(hits) == 2, f"Expected 2 indexed chunks, but got {len(hits)}"
for hit in hits:
source = hit.get("_source", {})
- assert source.get("document_id") == "sha-csv-integration-123"
+ assert source.get("document_id") == expected_document_id
assert source.get("filename") == "correlation_report.csv"
assert source.get("mimetype") == "text/csv"
assert "testlargephrase" in source.get("text", "")
diff --git a/tests/integration/core/test_onboarding_sample_docs.py b/tests/integration/core/test_onboarding_sample_docs.py
index 896e227cf..650f379b2 100644
--- a/tests/integration/core/test_onboarding_sample_docs.py
+++ b/tests/integration/core/test_onboarding_sample_docs.py
@@ -67,6 +67,10 @@ async def _require_langflow_ready() -> None:
async def isolated_onboarding_docs_workspace(tmp_path: Path, monkeypatch):
if not os.getenv("OPENAI_API_KEY"):
pytest.skip("OPENAI_API_KEY is required for onboarding sample-doc ingestion")
+ if not os.getenv("ANTHROPIC_API_KEY"):
+ pytest.skip(
+ "ANTHROPIC_API_KEY is required for onboarding sample-doc ingestion (ibm_anthropic.pdf)"
+ )
docs_dir = Path(__file__).resolve().parents[3] / "openrag-documents"
expected_filenames = sorted(
diff --git a/tests/unit/test_ascii_safe_header_value.py b/tests/unit/test_ascii_safe_header_value.py
new file mode 100644
index 000000000..2bf2ae746
--- /dev/null
+++ b/tests/unit/test_ascii_safe_header_value.py
@@ -0,0 +1,72 @@
+"""Pin the ASCII-safe HTTP header encoding used for Langflow global variables.
+
+A non-ASCII filename or owner name placed into an ``X-Langflow-Global-Var-*``
+header raised ``UnicodeEncodeError`` in httpx before the request was sent
+(httpx requires ASCII-encodable header values). The helper at
+`src/utils/langflow_headers.py::ascii_safe_header_value` percent-encodes only
+non-ASCII values; ASCII passes through untouched.
+"""
+
+import sys
+from pathlib import Path
+
+import httpx
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+from utils.langflow_headers import ascii_safe_header_value # noqa: E402
+
+
+ "value,expected",
+ [
+ # ASCII passes through byte-for-byte, including spaces and slashes.
+ ("report.pdf", "report.pdf"),
+ ("my report (final).pdf", "my report (final).pdf"),
+ ("a/b/c.txt", "a/b/c.txt"),
+ ("", ""),
+ # None coerces to empty string.
+ (None, ""),
+ # Non-string ASCII coerces via str().
+ (123, "123"),
+ ],
+)
+def test_ascii_values_pass_through(value, expected):
+ assert ascii_safe_header_value(value) == expected
+
+
+ "value",
+ [
+ "こんにちは こんにちは.pdf", # Japanese (the reported crash)
+ "José García.docx", # accented owner name
+ "файл.pdf", # Cyrillic
+ "emoji-📄.pdf", # emoji
+ ],
+)
+def test_non_ascii_values_become_ascii_encodable(value):
+ encoded = ascii_safe_header_value(value)
+ # The whole point: the result must survive httpx's ASCII encoding.
+ encoded.encode("ascii")
+ # It is percent-encoded, not silently dropped — the value is non-empty
+ # and differs from the raw input.
+ assert encoded
+ assert encoded != value
+
+
+def test_headers_dict_survives_httpx_normalization():
+ """Regression: a header dict carrying non-ASCII owner metadata must build
+ into httpx.Headers without raising (this is exactly where ingestion crashed).
+ X-Langflow-Global-Var-FILENAME was removed; OWNER_NAME/OWNER_EMAIL are the
+ remaining headers that can carry user-supplied non-ASCII values."""
+ headers = {
+ "X-Langflow-Global-Var-OWNER_NAME": ascii_safe_header_value("José García"),
+ "X-Langflow-Global-Var-OWNER_EMAIL": ascii_safe_header_value("josé@例え.jp"),
+ }
+ # Previously raised UnicodeEncodeError here.
+ httpx.Headers(headers)
diff --git a/tests/unit/test_langflow_ingest_callback.py b/tests/unit/test_langflow_ingest_callback.py
index fabf09e45..0f6236583 100644
--- a/tests/unit/test_langflow_ingest_callback.py
+++ b/tests/unit/test_langflow_ingest_callback.py
@@ -329,7 +329,8 @@ def test_ingest_flows_resolve_callback_config_from_global_vars(flow_path, compon
def test_ingest_flows_wire_callback_global_vars_into_opensearch(flow_path, component_id):
flow = json.loads(Path(flow_path).read_text(encoding="utf-8"))
nodes = {node.get("id"): node for node in flow["data"]["nodes"]}
- edges = flow["data"]["edges"]
+ component = nodes[component_id]
+ template = component["data"]["node"]["template"]
expected = {
"openrag_ingest_url": "OPENRAG_INGEST_URL",
"openrag_ingest_token": "OPENRAG_INGEST_TOKEN",
@@ -337,18 +338,14 @@ def test_ingest_flows_wire_callback_global_vars_into_opensearch(flow_path, compo
}
for field_name, variable_name in expected.items():
- source_id = f"TextInput-OpenRAGIngest-{field_name.removeprefix('openrag_ingest_')}"
- text_node = nodes[source_id]
- input_template = text_node["data"]["node"]["template"]["input_value"]
+ input_template = template[field_name]
assert input_template["value"] == variable_name
assert input_template["load_from_db"] is True
+ assert input_template["show"] is True
- assert any(
- edge.get("source") == source_id
- and edge.get("target") == component_id
- and edge.get("data", {}).get("targetHandle", {}).get("fieldName") == field_name
- for edge in edges
- )
+ assert template["openrag_ingest_batch_size"]["value"] == 100
+ assert template["openrag_ingest_batch_size"].get("load_from_db") is None
+ assert template["openrag_ingest_batch_size"]["show"] is True
@pytest.mark.parametrize(