OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Add cloud connector sync UI and selective backend
Edwin Josef23f98238d36
diff --git a/docker-compose.yml b/docker-compose.ymlindex fc11fe550..08b4018d9 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -128,6 +128,8 @@ services:- OWNER_EMAIL=None- CONNECTOR_TYPE=system- CONNECTOR_TYPE_URL=url+ - DOCUMENT_ID=+ - SOURCE_URL=- OPENRAG-QUERY-FILTER="{}"- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}- OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}@@ -138,7 +140,7 @@ services:- MIMETYPE=None- FILESIZE=0- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}- - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL+ - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL- LANGFLOW_LOG_LEVEL=DEBUG- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex 6f65c04e0..e00ff07cc 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -400,6 +400,62 @@"sourceHandle": "{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-EAo9iœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}","target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4","targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}"+ },+ {+ "animated": false,+ "data": {+ "sourceHandle": {+ "dataType": "SecretInput",+ "id": "SecretInput-b1QE5",+ "name": "text",+ "output_types": [+ "Message"+ ]+ },+ "targetHandle": {+ "fieldName": "dynamic_document_id",+ "id": "AdvancedDynamicFormBuilder-81Exw",+ "inputTypes": [+ "Text",+ "Message"+ ],+ "type": "str"+ }+ },+ "id": "xy-edge__SecretInput-b1QE5{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b1QE5œ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",+ "selected": false,+ "source": "SecretInput-b1QE5",+ "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b1QE5œ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",+ "target": "AdvancedDynamicFormBuilder-81Exw",+ "targetHandle": "{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"+ },+ {+ "animated": false,+ "data": {+ "sourceHandle": {+ "dataType": "SecretInput",+ "id": "SecretInput-KxF3a",+ "name": "text",+ "output_types": [+ "Message"+ ]+ },+ "targetHandle": {+ "fieldName": "dynamic_source_url",+ "id": "AdvancedDynamicFormBuilder-81Exw",+ "inputTypes": [+ "Text",+ "Message"+ ],+ "type": "str"+ }+ },+ "id": "xy-edge__SecretInput-KxF3a{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-KxF3aœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",+ "selected": false,+ "source": "SecretInput-KxF3a",+ "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-KxF3aœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",+ "target": "AdvancedDynamicFormBuilder-81Exw",+ "targetHandle": "{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"}],"nodes": [@@ -625,6 +681,7 @@"tool_mode": false},"selected_output": "chunks",+ "showNode": false,"type": "SplitText"},"dragging": false,@@ -635,8 +692,8 @@"width": 320},"position": {- "x": 1711.934915237861,- "y": 1637.2034518030887+ "x": 1354.9418688156868,+ "y": 1768.0815618919698},"positionAbsolute": {"x": 1683.4543896546102,@@ -667,7 +724,7 @@],"frozen": false,"icon": "braces",- "last_updated": "2025-12-12T20:12:18.129Z",+ "last_updated": "2026-02-06T18:04:39.034Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {},@@ -717,7 +774,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","code": {@@ -771,6 +828,39 @@"type": "str","value": ""},+ "dynamic_document_id": {+ "_input_type": "MultilineInput",+ "advanced": false,+ "ai_enabled": false,+ "copy_field": false,+ "display_name": "document_id",+ "dynamic": false,+ "helper_text": null,+ "info": " (Can connect to: Text, Message)",+ "input_types": [+ "Text",+ "Message"+ ],+ "list": false,+ "list_add_label": "Add More",+ "load_from_db": false,+ "multiline": true,+ "name": "dynamic_document_id",+ "override_skip": false,+ "placeholder": "",+ "real_time_refresh": null,+ "refresh_button": null,+ "refresh_button_text": null,+ "required": false,+ "show": true,+ "title_case": false,+ "tool_mode": false,+ "trace_as_input": true,+ "trace_as_metadata": true,+ "track_in_telemetry": false,+ "type": "str",+ "value": ""+ },"dynamic_owner": {"_input_type": "MultilineInput","advanced": false,@@ -870,6 +960,39 @@"type": "str","value": ""},+ "dynamic_source_url": {+ "_input_type": "MultilineInput",+ "advanced": false,+ "ai_enabled": false,+ "copy_field": false,+ "display_name": "source_url",+ "dynamic": false,+ "helper_text": null,+ "info": " (Can connect to: Text, Message)",+ "input_types": [+ "Text",+ "Message"+ ],+ "list": false,+ "list_add_label": "Add More",+ "load_from_db": false,+ "multiline": true,+ "name": "dynamic_source_url",+ "override_skip": false,+ "placeholder": "",+ "real_time_refresh": null,+ "refresh_button": null,+ "refresh_button_text": null,+ "required": false,+ "show": true,+ "title_case": false,+ "tool_mode": false,+ "trace_as_input": true,+ "trace_as_metadata": true,+ "track_in_telemetry": false,+ "type": "str",+ "value": ""+ },"form_fields": {"_input_type": "TableInput","advanced": false,@@ -944,6 +1067,14 @@{"field_name": "connector_type","field_type": "Text"+ },+ {+ "field_name": "source_url",+ "field_type": "Text"+ },+ {+ "field_name": "document_id",+ "field_type": "Text"}]},@@ -970,18 +1101,18 @@"tool_mode": false},"selected_output": "form_data",- "showNode": true,+ "showNode": false,"type": "AdvancedDynamicFormBuilder"},"dragging": false,"id": "AdvancedDynamicFormBuilder-81Exw","measured": {- "height": 552,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1778.514096901592,- "y": 673.5870187063417+ "x": 1352.0744672643646,+ "y": 1508.919401750829},"selected": false,"type": "genericNode"@@ -1070,18 +1201,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "SecretInput"},"dragging": false,"id": "SecretInput-F34VJ","measured": {- "height": 220,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1343.2266447254406,- "y": 503.74766485111434+ "x": 1089.1452135742702,+ "y": 1355.8795635459912},"selected": false,"type": "genericNode"@@ -1170,18 +1301,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "SecretInput"},"dragging": false,"id": "SecretInput-b2cab","measured": {- "height": 220,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1341.4694747797632,- "y": 753.9209691638071+ "x": 1085.2410189000832,+ "y": 1484.8702711989886},"selected": false,"type": "genericNode"@@ -1270,18 +1401,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "SecretInput"},"dragging": false,"id": "SecretInput-ZVfuS","measured": {- "height": 220,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1336.2669044250051,- "y": 1000.9543699805594+ "x": 1083.5090209646805,+ "y": 1549.912686959435},"selected": false,"type": "genericNode"@@ -1370,18 +1501,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "SecretInput"},"dragging": false,"id": "SecretInput-Iqtxd","measured": {- "height": 220,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1342.0034534756019,- "y": 1239.4741232342342+ "x": 1084.150007027148,+ "y": 1616.4708779408236},"selected": false,"type": "genericNode"@@ -1496,7 +1627,7 @@"info": "URL of the Docling Serve instance.","list": false,"list_add_label": "Add More",- "load_from_db": true,+ "load_from_db": false,"name": "api_url","override_skip": false,"placeholder": "",@@ -1507,7 +1638,7 @@"trace_as_metadata": true,"track_in_telemetry": false,"type": "str",- "value": "DOCLING_SERVE_URL"+ "value": "http://localhost:5001"},"code": {"advanced": true,@@ -1785,18 +1916,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "DoclingRemote"},"dragging": false,"id": "DoclingRemote-Dp3PX","measured": {- "height": 312,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": -175.49741984154275,- "y": 1505.0245107748437+ "x": -27.54468027620713,+ "y": 1770.6264167415306},"selected": false,"type": "genericNode"@@ -2042,18 +2173,18 @@"tool_mode": false},"selected_output": "dataframe",- "showNode": true,+ "showNode": false,"type": "ExportDoclingDocument"},"dragging": false,"id": "ExportDoclingDocument-zZdRg","measured": {- "height": 347,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 206.97755086947473,- "y": 1610.6498167995744+ "x": 258.9758629312913,+ "y": 1768.6446880643248},"selected": false,"type": "genericNode"@@ -2090,7 +2221,7 @@],"frozen": false,"icon": "table",- "last_updated": "2025-12-12T20:12:18.208Z",+ "last_updated": "2026-02-06T17:43:52.579Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -2137,7 +2268,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","ascending": {@@ -2493,18 +2624,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "DataFrameOperations"},"dragging": false,"id": "DataFrameOperations-1BWXB","measured": {- "height": 399,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 575.218683966709,- "y": 1607.3264908868193+ "x": 533.2200473013957,+ "y": 1769.3212323101711},"selected": false,"type": "genericNode"@@ -2541,7 +2672,7 @@],"frozen": false,"icon": "table",- "last_updated": "2025-12-12T20:12:18.209Z",+ "last_updated": "2026-02-06T17:43:52.579Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -2588,7 +2719,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","ascending": {@@ -2944,18 +3075,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "DataFrameOperations"},"dragging": false,"id": "DataFrameOperations-N80fC","measured": {- "height": 399,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1330.149865256777,- "y": 1619.9268393528012+ "x": 1070.158304947694,+ "y": 1767.9220352210484},"selected": false,"type": "genericNode"@@ -2992,7 +3123,7 @@],"frozen": false,"icon": "table",- "last_updated": "2025-12-12T20:12:18.209Z",+ "last_updated": "2026-02-06T17:43:52.580Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -3039,7 +3170,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","ascending": {@@ -3395,18 +3526,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "DataFrameOperations"},"dragging": false,"id": "DataFrameOperations-9vMrp","measured": {- "height": 399,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 937.1310281139399,- "y": 1611.2186890450444+ "x": 787.135897166392,+ "y": 1770.3912596804048},"selected": false,"type": "genericNode"@@ -3430,8 +3561,8 @@"width": 1000},"position": {- "x": -184.83853691310878,- "y": 944.7352701051674+ "x": -21.449808371464087,+ "y": 1292.5429595750052},"resizing": true,"selected": false,@@ -3950,7 +4081,7 @@"info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).","list": false,"list_add_label": "Add More",- "load_from_db": true,+ "load_from_db": false,"name": "opensearch_url","override_skip": false,"placeholder": "",@@ -3961,7 +4092,7 @@"trace_as_metadata": true,"track_in_telemetry": false,"type": "str",- "value": "OPENSEARCH_URL"+ "value": "https://opensearch:9200"},"password": {"_input_type": "SecretStrInput",@@ -4143,18 +4274,18 @@"tool_mode": false},"selected_output": "search_results",- "showNode": true,+ "showNode": false,"type": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding"},"dragging": false,"id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4","measured": {- "height": 904,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 2261.865622928042,- "y": 1349.2821108833643+ "x": 1906.1233004091807,+ "y": 1854.130454249347},"selected": false,"type": "genericNode"@@ -4193,7 +4324,7 @@],"frozen": false,"icon": "binary",- "last_updated": "2025-12-12T20:12:18.131Z",+ "last_updated": "2026-02-06T18:05:46.249Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -4261,13 +4392,13 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","api_base": {"_input_type": "MessageTextInput",- "advanced": true,- "display_name": "API Base URL",+ "advanced": false,+ "display_name": "OpenAI API Base URL","dynamic": false,"info": "Base URL for the API. Leave empty for default.","input_types": [@@ -4280,7 +4411,7 @@"override_skip": false,"placeholder": "","required": false,- "show": false,+ "show": true,"title_case": false,"tool_mode": false,"trace_as_input": true,@@ -4292,7 +4423,7 @@"api_key": {"_input_type": "SecretStrInput","advanced": false,- "display_name": "IBM watsonx.ai API Key",+ "display_name": "OpenAI API Key","dynamic": false,"info": "Model Provider API key","input_types": [],@@ -4307,7 +4438,7 @@"title_case": false,"track_in_telemetry": false,"type": "str",- "value": "WATSONX_API_KEY"+ "value": "OPENAI_API_KEY"},"base_url_ibm_watsonx": {"_input_type": "DropdownInput",@@ -4332,7 +4463,7 @@"placeholder": "","real_time_refresh": true,"required": false,- "show": true,+ "show": false,"title_case": false,"toggle": false,"tool_mode": false,@@ -4433,7 +4564,7 @@"override_skip": false,"placeholder": "","required": false,- "show": true,+ "show": false,"title_case": false,"tool_mode": false,"trace_as_metadata": true,@@ -4474,7 +4605,7 @@"load_from_db": false,"name": "model","options": [- "ibm/granite-embedding-278m-multilingual"+ "text-embedding-3-small"],"options_metadata": [],"override_skip": false,@@ -4489,7 +4620,7 @@"trace_as_metadata": true,"track_in_telemetry": true,"type": "str",- "value": "ibm/granite-embedding-278m-multilingual"+ "value": "text-embedding-3-small"},"model_kwargs": {"_input_type": "DictInput",@@ -4553,7 +4684,7 @@"override_skip": false,"placeholder": "","required": false,- "show": true,+ "show": false,"title_case": false,"tool_mode": false,"trace_as_input": true,@@ -4603,7 +4734,7 @@"trace_as_metadata": true,"track_in_telemetry": true,"type": "str",- "value": "IBM watsonx.ai"+ "value": "OpenAI"},"request_timeout": {"_input_type": "FloatInput",@@ -4657,7 +4788,7 @@"override_skip": false,"placeholder": "","required": false,- "show": true,+ "show": false,"title_case": false,"tool_mode": false,"trace_as_metadata": true,@@ -4668,18 +4799,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "EmbeddingModel"},"dragging": false,"id": "EmbeddingModel-EAo9i","measured": {- "height": 534,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 838.9563350647003,- "y": 2191.523005861695+ "x": 1355.5927560779853,+ "y": 1905.3991707633513},"selected": false,"type": "genericNode"@@ -4718,7 +4849,7 @@],"frozen": false,"icon": "binary",- "last_updated": "2025-12-12T20:12:18.132Z",+ "last_updated": "2026-02-06T18:05:43.252Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -4786,7 +4917,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","api_base": {@@ -5194,18 +5325,18 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "EmbeddingModel"},"dragging": false,"id": "EmbeddingModel-E0hvR","measured": {- "height": 369,- "width": 320+ "height": 48,+ "width": 192},"position": {- "x": 1223.4804629271505,- "y": 2198.8989246514284+ "x": 1361.1152821119326,+ "y": 1983.3161387480468},"selected": false,"type": "genericNode"@@ -5244,7 +5375,7 @@],"frozen": false,"icon": "binary",- "last_updated": "2025-12-12T20:12:18.133Z",+ "last_updated": "2026-02-06T18:05:39.873Z","legacy": false,"lf_version": "1.7.0.dev21","metadata": {@@ -5312,7 +5443,7 @@"value": "5488df7c-b93f-4f87-a446-b67028bc0813"},"_frontend_node_folder_id": {- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"},"_type": "Component","api_base": {@@ -5719,27 +5850,227 @@},"tool_mode": false},- "showNode": true,+ "showNode": false,"type": "EmbeddingModel"},"dragging": false,"id": "EmbeddingModel-3LsIP","measured": {- "height": 369,- "width": 320+ "height": 48,+ "width": 192+ },+ "position": {+ "x": 1360.2396840279503,+ "y": 2067.099324740414+ },+ "selected": false,+ "type": "genericNode"+ },+ {+ "data": {+ "id": "SecretInput-b1QE5",+ "node": {+ "base_classes": [+ "Message"+ ],+ "beta": false,+ "conditional_paths": [],+ "custom_fields": {},+ "description": "Allows the selection of a secret to be generated as output..",+ "display_name": "Secret Input",+ "documentation": "https://docs.langflow.org/components-io#text-input",+ "edited": true,+ "field_order": [+ "input_value"+ ],+ "frozen": false,+ "icon": "type",+ "legacy": false,+ "lf_version": "1.7.0.dev21",+ "metadata": {},+ "minimized": false,+ "output_types": [],+ "outputs": [+ {+ "allows_loop": false,+ "cache": true,+ "display_name": "Output Text",+ "group_outputs": false,+ "hidden": null,+ "method": "text_response",+ "name": "text",+ "options": null,+ "required_inputs": null,+ "selected": "Message",+ "tool_mode": true,+ "types": [+ "Message"+ ],+ "value": "__UNDEFINED__"+ }+ ],+ "pinned": false,+ "template": {+ "_type": "Component",+ "code": {+ "advanced": true,+ "dynamic": true,+ "fileTypes": [],+ "file_path": "",+ "info": "",+ "list": false,+ "load_from_db": false,+ "multiline": true,+ "name": "code",+ "password": false,+ "placeholder": "",+ "required": true,+ "show": true,+ "title_case": false,+ "type": "code",+ "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n"+ },+ "input_value": {+ "_input_type": "SecretStrInput",+ "advanced": false,+ "display_name": "Secret",+ "dynamic": false,+ "info": "Secret to be passed as input.",+ "input_types": [],+ "load_from_db": true,+ "name": "input_value",+ "password": true,+ "placeholder": "",+ "required": false,+ "show": true,+ "title_case": false,+ "type": "str",+ "value": "DOCUMENT_ID"+ }+ },+ "tool_mode": false+ },+ "showNode": false,+ "type": "SecretInput"+ },+ "dragging": false,+ "id": "SecretInput-b1QE5",+ "measured": {+ "height": 48,+ "width": 192+ },+ "position": {+ "x": 1085.4357429636027,+ "y": 1419.65527860865+ },+ "selected": false,+ "type": "genericNode"+ },+ {+ "data": {+ "id": "SecretInput-KxF3a",+ "node": {+ "base_classes": [+ "Message"+ ],+ "beta": false,+ "conditional_paths": [],+ "custom_fields": {},+ "description": "Allows the selection of a secret to be generated as output..",+ "display_name": "Secret Input",+ "documentation": "https://docs.langflow.org/components-io#text-input",+ "edited": true,+ "field_order": [+ "input_value"+ ],+ "frozen": false,+ "icon": "type",+ "legacy": false,+ "lf_version": "1.7.0.dev21",+ "metadata": {},+ "minimized": false,+ "output_types": [],+ "outputs": [+ {+ "allows_loop": false,+ "cache": true,+ "display_name": "Output Text",+ "group_outputs": false,+ "hidden": null,+ "method": "text_response",+ "name": "text",+ "options": null,+ "required_inputs": null,+ "selected": "Message",+ "tool_mode": true,+ "types": [+ "Message"+ ],+ "value": "__UNDEFINED__"+ }+ ],+ "pinned": false,+ "template": {+ "_type": "Component",+ "code": {+ "advanced": true,+ "dynamic": true,+ "fileTypes": [],+ "file_path": "",+ "info": "",+ "list": false,+ "load_from_db": false,+ "multiline": true,+ "name": "code",+ "password": false,+ "placeholder": "",+ "required": true,+ "show": true,+ "title_case": false,+ "type": "code",+ "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n"+ },+ "input_value": {+ "_input_type": "SecretStrInput",+ "advanced": false,+ "display_name": "Secret",+ "dynamic": false,+ "info": "Secret to be passed as input.",+ "input_types": [],+ "load_from_db": true,+ "name": "input_value",+ "password": true,+ "placeholder": "",+ "required": false,+ "show": true,+ "title_case": false,+ "type": "str",+ "value": "SOURCE_URL"+ }+ },+ "tool_mode": false+ },+ "showNode": false,+ "type": "SecretInput"+ },+ "dragging": false,+ "id": "SecretInput-KxF3a",+ "measured": {+ "height": 48,+ "width": 192},"position": {- "x": 1638.9179466145608,- "y": 2110.0422159522327+ "x": 1085.9081887151694,+ "y": 1680.6730861389885},"selected": false,"type": "genericNode"}],"viewport": {- "x": 249.3666737262397,- "y": -156.8776378758762,- "zoom": 0.38977017930844676+ "x": 47.599403481811805,+ "y": -629.4207733468738,+ "zoom": 0.6293990172646186}},"description": "Load your data for chat context with Retrieval Augmented Generation.",diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsnew file mode 100644index 000000000..e87f92a17--- /dev/null+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -0,0 +1,77 @@+"use client";++import { useMutation, useQueryClient } from "@tanstack/react-query";++// Response types+interface SyncResponse {+ task_ids?: string[];+ status: string;+ message: string;+ connections_synced?: number;+ synced_connectors?: string[];+ skipped_connectors?: string[];+ errors?: Array<{ connector_type: string; error: string }> | null;+}++// Sync all cloud connectors+const syncAllConnectors = async (): Promise<SyncResponse> => {+ const response = await fetch("/api/connectors/sync-all", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ });++ if (!response.ok) {+ const error = await response.json();+ throw new Error(error.error || "Failed to sync connectors");+ }++ return response.json();+};++// Sync a specific connector type+const syncConnector = async (connectorType: string): Promise<SyncResponse> => {+ const response = await fetch(`/api/connectors/${connectorType}/sync`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({}),+ });++ if (!response.ok) {+ const error = await response.json();+ throw new Error(error.error || `Failed to sync ${connectorType}`);+ }++ return response.json();+};++export const useSyncAllConnectors = () => {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: syncAllConnectors,+ onSettled: () => {+ // Invalidate and refetch search queries after a delay to allow sync to complete+ setTimeout(() => {+ queryClient.invalidateQueries({ queryKey: ["search"] });+ }, 2000);+ },+ });+};++export const useSyncConnector = () => {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: syncConnector,+ onSettled: () => {+ // Invalidate and refetch search queries after a delay to allow sync to complete+ setTimeout(() => {+ queryClient.invalidateQueries({ queryKey: ["search"] });+ }, 2000);+ },+ });+};diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex 85189a85f..0277a3344 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -8,7 +8,7 @@ import {type ValueFormatterParams,} from "ag-grid-community";import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";-import { Cloud, FileIcon, Globe } from "lucide-react";+import { Cloud, FileIcon, Globe, RefreshCw } from "lucide-react";import { useRouter } from "next/navigation";import { useCallback, useEffect, useRef, useState } from "react";import { KnowledgeDropdown } from "@/components/knowledge-dropdown";@@ -44,6 +44,7 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";import OneDriveIcon from "../../components/icons/one-drive-logo";import SharePointIcon from "../../components/icons/share-point-logo";import { useDeleteDocument } from "../api/mutations/useDeleteDocument";+import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";// Function to get the appropriate icon for a connector typefunction getSourceIcon(connectorType?: string) {@@ -78,6 +79,7 @@ function SearchPage() {const lastErrorRef = useRef<string | null>(null);const deleteDocumentMutation = useDeleteDocument();+ const syncAllConnectorsMutation = useSyncAllConnectors();useEffect(() => {refreshTasks();@@ -297,7 +299,12 @@ function SearchPage() {if (status !== "active") {return null;}- return <KnowledgeActionsDropdown filename={data?.filename || ""} />;+ return (+ <KnowledgeActionsDropdown+ filename={data?.filename || ""}+ connectorType={data?.connector_type}+ />+ );},cellStyle: {alignItems: "center",@@ -380,15 +387,43 @@ function SearchPage() {{/* Search Input Area */}<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6"><KnowledgeSearchInput />- {/* //TODO: Implement sync button */}- {/* <Button- type="button"- variant="outline"- className="rounded-lg flex-shrink-0"- onClick={() => alert("Not implemented")}- >- Sync- </Button> */}+ <Button+ type="button"+ variant="outline"+ className="rounded-lg flex-shrink-0"+ disabled={syncAllConnectorsMutation.isPending}+ onClick={async () => {+ try {+ toast.info("Syncing all cloud connectors...");+ const result = await syncAllConnectorsMutation.mutateAsync();+ if (result.status === "no_files") {+ toast.info(result.message || "No cloud files to sync. Add files from cloud connectors first.");+ } else if (result.synced_connectors && result.synced_connectors.length > 0) {+ toast.success(+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`+ );+ } else if (result.errors && result.errors.length > 0) {+ toast.error("Some connectors failed to sync");+ }+ } catch (error) {+ toast.error(+ error instanceof Error ? error.message : "Failed to sync connectors"+ );+ }+ }}+ >+ {syncAllConnectorsMutation.isPending ? (+ <>+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />+ Syncing...+ </>+ ) : (+ <>+ <RefreshCw className="h-4 w-4 mr-2" />+ Sync+ </>+ )}+ </Button>{selectedRows.length > 0 && (<Buttontype="button"diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsxindex a144c8ac3..45eae1e36 100644--- a/frontend/components/knowledge-actions-dropdown.tsx+++ b/frontend/components/knowledge-actions-dropdown.tsx@@ -1,10 +1,11 @@"use client";-import { EllipsisVertical } from "lucide-react";+import { EllipsisVertical, RefreshCw } from "lucide-react";import { useRouter } from "next/navigation";import { useState } from "react";import { toast } from "sonner";import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";+import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import {DropdownMenu,DropdownMenuContent,@@ -16,15 +17,24 @@ import { Button } from "./ui/button";interface KnowledgeActionsDropdownProps {filename: string;+ connectorType?: string;}+// Cloud connector types that support sync+const CLOUD_CONNECTOR_TYPES = new Set(["google_drive", "onedrive", "sharepoint"]);+export const KnowledgeActionsDropdown = ({filename,+ connectorType,}: KnowledgeActionsDropdownProps) => {const [showDeleteDialog, setShowDeleteDialog] = useState(false);const deleteDocumentMutation = useDeleteDocument();+ const syncConnectorMutation = useSyncConnector();const router = useRouter();+ // Check if this file is from a cloud connector (can be synced)+ const isCloudFile = connectorType && CLOUD_CONNECTOR_TYPES.has(connectorType);+const handleDelete = async () => {try {await deleteDocumentMutation.mutateAsync({ filename });@@ -37,6 +47,26 @@ export const KnowledgeActionsDropdown = ({}};+ const handleSync = async () => {+ if (!connectorType) return;++ try {+ toast.info(`Syncing ${connectorType}...`);+ const result = await syncConnectorMutation.mutateAsync(connectorType);+ if (result.status === "no_files") {+ toast.info(result.message || `No ${connectorType} files to sync.`);+ } else if (result.task_ids && result.task_ids.length > 0) {+ toast.success(+ `Sync started for ${connectorType}. Check task notifications for progress.`+ );+ }+ } catch (error) {+ toast.error(+ error instanceof Error ? error.message : `Failed to sync ${connectorType}`,+ );+ }+ };+return (<><DropdownMenu>@@ -56,19 +86,25 @@ export const KnowledgeActionsDropdown = ({>View chunks</DropdownMenuItem>- {/* //TODO: Implement rename and sync */}- {/* <DropdownMenuItem- className="text-primary focus:text-primary"- onClick={() => alert("Not implemented")}- >- Rename- </DropdownMenuItem>- <DropdownMenuItem- className="text-primary focus:text-primary"- onClick={() => alert("Not implemented")}- >- Sync- </DropdownMenuItem> */}+ {isCloudFile && (+ <DropdownMenuItem+ className="text-primary focus:text-primary"+ disabled={syncConnectorMutation.isPending}+ onClick={handleSync}+ >+ {syncConnectorMutation.isPending ? (+ <>+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />+ Syncing...+ </>+ ) : (+ <>+ <RefreshCw className="h-4 w-4 mr-2" />+ Sync+ </>+ )}+ </DropdownMenuItem>+ )}<DropdownMenuItemclassName="text-destructive focus:text-destructive"onClick={() => setShowDeleteDialog(true)}diff --git a/helm/openrag/templates/langflow/deployment.yaml b/helm/openrag/templates/langflow/deployment.yamlindex 2683011f4..377235401 100644--- a/helm/openrag/templates/langflow/deployment.yaml+++ b/helm/openrag/templates/langflow/deployment.yaml@@ -100,6 +100,10 @@ spec:value: "system"- name: CONNECTOR_TYPE_URLvalue: "url"+ - name: DOCUMENT_ID+ value: ""+ - name: SOURCE_URL+ value: ""- name: OPENRAG-QUERY-FILTERvalue: "{}"- name: FILENAMEdiff --git a/helm/openrag/values.yaml b/helm/openrag/values.yamlindex fbf0707a3..2a2b52371 100644--- a/helm/openrag/values.yaml+++ b/helm/openrag/values.yaml@@ -92,7 +92,7 @@ langflow:logLevel: "INFO" # DEBUG, INFO, WARNING, ERROR# Variables to expose to flows- variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL"+ variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL"# ProbeslivenessProbe:diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 9824e6a71..4c54097c7 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -2,10 +2,82 @@ from starlette.requests import Requestfrom starlette.responses import JSONResponse, PlainTextResponsefrom utils.logging_config import get_loggerfrom utils.telemetry import TelemetryClient, Category, MessageId+from config.settings import INDEX_NAMElogger = get_logger(__name__)+async def get_synced_file_ids_for_connector(+ connector_type: str,+ user_id: str,+ session_manager,+ jwt_token: str = None,+) -> tuple:+ """+ Query OpenSearch for unique document_id values where connector_type matches.+ Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.++ Note: Langflow-ingested files may not have document_id stored. In that case,+ filenames are returned for filename-based filtering during sync.+ """+ try:+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)++ # Query for both document_id and filename aggregations+ query_body = {+ "size": 0,+ "query": {+ "term": {+ "connector_type": connector_type+ }+ },+ "aggs": {+ "unique_document_ids": {+ "terms": {+ "field": "document_id",+ "size": 10000+ }+ },+ "unique_filenames": {+ "terms": {+ "field": "filename",+ "size": 10000+ }+ }+ }+ }++ result = await opensearch_client.search(+ index=INDEX_NAME,+ body=query_body+ )++ # Get document_ids (preferred - these are the actual connector file IDs)+ doc_id_buckets = result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])+ file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]++ # Get filenames as fallback+ filename_buckets = result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])+ filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]++ logger.debug(+ "Found synced files for connector",+ connector_type=connector_type,+ file_ids_count=len(file_ids),+ filenames_count=len(filenames),+ )++ return file_ids, filenames++ except Exception as e:+ logger.error(+ "Failed to get synced file IDs",+ connector_type=connector_type,+ error=str(e),+ )+ return [], []++async def list_connectors(request: Request, connector_service, session_manager):"""List available connector types with metadata"""try:@@ -104,6 +176,7 @@ async def connector_sync(request: Request, connector_service, session_manager):)if selected_files:+ # Explicit files selected (e.g., from file picker) - sync those specific filestask_id = await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,@@ -112,12 +185,52 @@ async def connector_sync(request: Request, connector_service, session_manager):file_infos=file_infos,)else:- task_id = await connector_service.sync_connector_files(- working_connection.connection_id,- user.user_id,- max_files,+ # No files specified - sync only files already in OpenSearch for this connector+ # This ensures deleted files stay deleted+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,jwt_token=jwt_token,)++ if not existing_file_ids and not existing_filenames:+ return JSONResponse(+ {+ "status": "no_files",+ "message": f"No {connector_type} files to sync. Add files from the connector first.",+ },+ status_code=200,+ )++ # If we have document_ids (connector file IDs), use sync_specific_files+ # Otherwise, use filename filtering with sync_connector_files+ if existing_file_ids:+ logger.info(+ "Syncing specific files by document_id",+ connector_type=connector_type,+ file_count=len(existing_file_ids),+ )+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ existing_file_ids,+ jwt_token=jwt_token,+ )+ else:+ # Fallback: use filename filtering (for Langflow-ingested files without document_id)+ logger.info(+ "Syncing files by filename filter (document_id not available)",+ connector_type=connector_type,+ filename_count=len(existing_filenames),+ )+ task_id = await connector_service.sync_connector_files(+ working_connection.connection_id,+ user.user_id,+ max_files=None,+ jwt_token=jwt_token,+ filename_filter=set(existing_filenames),+ )task_ids = [task_id]await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)return JSONResponse(@@ -467,6 +580,159 @@ async def connector_disconnect(request: Request, connector_service, session_mana)+async def sync_all_connectors(request: Request, connector_service, session_manager):+ """+ Sync files from all active cloud connector connections (Google Drive, OneDrive, SharePoint).++ Only syncs files that are already indexed in OpenSearch - this ensures deleted files+ stay deleted and only previously selected files get re-synced (to update content and ACLs).+ """+ try:+ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)+ user = request.state.user+ jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)++ # Cloud connector types to sync+ cloud_connector_types = ["google_drive", "onedrive", "sharepoint"]++ all_task_ids = []+ synced_connectors = []+ skipped_connectors = []+ errors = []++ for connector_type in cloud_connector_types:+ try:+ # First, get existing file IDs/filenames from OpenSearch for this connector type+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )++ if not existing_file_ids and not existing_filenames:+ logger.debug(+ "No existing files in OpenSearch for connector type, skipping",+ connector_type=connector_type,+ )+ skipped_connectors.append(connector_type)+ continue++ # Get all active connections for this connector type and user+ connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type=connector_type+ )++ active_connections = [conn for conn in connections if conn.is_active]+ if not active_connections:+ logger.debug(+ "No active connections for connector type",+ connector_type=connector_type,+ )+ continue++ # Find the first connection that actually works+ working_connection = None+ for connection in active_connections:+ try:+ connector = await connector_service.get_connector(connection.connection_id)+ if connector and await connector.authenticate():+ working_connection = connection+ break+ except Exception as e:+ logger.debug(+ "Connection validation failed",+ connection_id=connection.connection_id,+ error=str(e),+ )+ continue++ if not working_connection:+ logger.debug(+ "No working connection for connector type",+ connector_type=connector_type,+ )+ continue++ # Sync using document_ids if available, else use filename filter+ if existing_file_ids:+ logger.info(+ "Syncing specific files by document_id",+ connector_type=connector_type,+ file_count=len(existing_file_ids),+ )+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ existing_file_ids,+ jwt_token=jwt_token,+ )+ else:+ # Fallback: use filename filtering+ logger.info(+ "Syncing files by filename filter",+ connector_type=connector_type,+ filename_count=len(existing_filenames),+ )+ task_id = await connector_service.sync_connector_files(+ working_connection.connection_id,+ user.user_id,+ max_files=None,+ jwt_token=jwt_token,+ filename_filter=set(existing_filenames),+ )++ all_task_ids.append(task_id)+ synced_connectors.append(connector_type)+ logger.info(+ "Started sync for connector type",+ connector_type=connector_type,+ task_id=task_id,+ file_count=len(existing_file_ids) if existing_file_ids else len(existing_filenames),+ )++ except Exception as e:+ logger.error(+ "Failed to sync connector type",+ connector_type=connector_type,+ error=str(e),+ )+ errors.append({"connector_type": connector_type, "error": str(e)})++ if not all_task_ids and not errors:+ if skipped_connectors:+ return JSONResponse(+ {+ "status": "no_files",+ "message": "No files to sync. Add files from cloud connectors first.",+ "skipped_connectors": skipped_connectors,+ },+ status_code=200,+ )+ return JSONResponse(+ {"error": "No active cloud connector connections found"},+ status_code=404,+ )++ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)+ return JSONResponse(+ {+ "task_ids": all_task_ids,+ "status": "sync_started",+ "message": f"Started syncing files from {len(synced_connectors)} cloud connector(s)",+ "synced_connectors": synced_connectors,+ "skipped_connectors": skipped_connectors if skipped_connectors else None,+ "errors": errors if errors else None,+ },+ status_code=201,+ )++ except Exception as e:+ logger.error("Sync all connectors failed", error=str(e))+ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)+ return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)++async def connector_token(request: Request, connector_service, session_manager):"""Get access token for connector API calls (e.g., Pickers)."""url_connector_type = request.path_params.get("connector_type")diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 49be8588b..3d5eeda5d 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -64,12 +64,26 @@ class LangflowConnectorService:# Step 1: Upload file to Langflowlogger.debug("Uploading file to Langflow", filename=document.filename)content = document.content+ processed_filename = document.filename.replace(" ", "_").replace("/", "_") + suffixfile_tuple = (- document.filename.replace(" ", "_").replace("/", "_") + suffix,+ processed_filename,content,document.mimetype or "application/octet-stream",)+ # Step 0: Delete existing chunks for this file before re-ingesting+ # This prevents duplicate chunks when syncing files+ if self.session_manager:+ try:+ from config.settings import INDEX_NAME+ opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token)+ delete_body = {"query": {"term": {"filename": processed_filename}}}+ delete_result = await opensearch_client.delete_by_query(index=INDEX_NAME, body=delete_body)+ deleted_count = delete_result.get("deleted", 0)+ logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count)+ except Exception as delete_err:+ logger.warning("Failed to delete existing chunks before re-ingestion", filename=processed_filename, error=str(delete_err))+langflow_file_id = None # Initialize to track if upload succeededtry:upload_result = await self.langflow_service.upload_user_file(@@ -101,6 +115,8 @@ class LangflowConnectorService:owner_name=owner_name,owner_email=owner_email,connector_type=connector_type,+ document_id=document.id,+ source_url=document.source_url,)logger.debug("Ingestion flow completed", result=ingestion_result)diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 64c74f5cf..2bc6e9986 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -200,8 +200,20 @@ class ConnectorService:user_id: str,max_files: int = None,jwt_token: str = None,+ filename_filter: set = None,) -> str:- """Sync files from a connector connection using existing task tracking system"""+ """+ Sync files from a connector connection using existing task tracking system.++ Args:+ connection_id: The connection ID+ user_id: The user ID+ max_files: Maximum number of files to sync+ jwt_token: Optional JWT token+ filename_filter: Optional set of filenames to filter - only files with names+ in this set will be synced. Used to prevent deleted files+ from being re-synced.+ """if not self.task_service:raise ValueError("TaskService not available - connector sync requires task service dependency"@@ -248,6 +260,15 @@ class ConnectorService:for file_info in files:if max_files and len(files_to_process) >= max_files:break+ # Filter by filename if filter is provided+ if filename_filter is not None:+ file_name = file_info.get("name", "")+ if file_name not in filename_filter:+ logger.debug(+ "Skipping file not in filter",+ filename=file_name,+ )+ continuefiles_to_process.append(file_info)# Stop if we have enough files or no more pagesdiff --git a/src/main.py b/src/main.pyindex 6c8b696b2..60903746f 100644--- a/src/main.py+++ b/src/main.py@@ -1077,6 +1077,17 @@ async def create_app():),methods=["POST"],),+ Route(+ "/connectors/sync-all",+ require_auth(services["session_manager"])(+ partial(+ connectors.sync_all_connectors,+ connector_service=services["connector_service"],+ session_manager=services["session_manager"],+ )+ ),+ methods=["POST"],+ ),Route("/connectors/{connector_type}/status",require_auth(services["session_manager"])(diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex ca5fec7ca..e670e2b26 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -68,6 +68,8 @@ class LangflowFileService:owner_name: Optional[str] = None,owner_email: Optional[str] = None,connector_type: Optional[str] = None,+ document_id: Optional[str] = None,+ source_url: Optional[str] = None,) -> Dict[str, Any]:"""Trigger the ingestion flow with provided file paths.@@ -157,6 +159,8 @@ class LangflowFileService:"X-Langflow-Global-Var-MIMETYPE": mimetype,"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),+ "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id) if document_id else "",+ "X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "",}# Add provider credentials as global variables for ingestion