Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Add cloud connector sync UI and selective backend

Edwin Josef23f98238d36
diff --git a/docker-compose.yml b/docker-compose.yml
index fc11fe550..08b4018d9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -128,6 +128,8 @@ services:
- OWNER_EMAIL=None
- CONNECTOR_TYPE=system
- CONNECTOR_TYPE_URL=url
+ - DOCUMENT_ID=
+ - SOURCE_URL=
- OPENRAG-QUERY-FILTER="{}"
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
@@ -138,7 +140,7 @@ services:
- MIMETYPE=None
- FILESIZE=0
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}
- - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL
+ - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL
- LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json
index 6f65c04e0..e00ff07cc 100644
--- a/flows/ingestion_flow.json
+++ b/flows/ingestion_flow.json
@@ -400,6 +400,62 @@
"sourceHandle": "{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-EAo9iœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}",
"target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
"targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}"
+ },
+ {
+ "animated": false,
+ "data": {
+ "sourceHandle": {
+ "dataType": "SecretInput",
+ "id": "SecretInput-b1QE5",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "dynamic_document_id",
+ "id": "AdvancedDynamicFormBuilder-81Exw",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__SecretInput-b1QE5{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b1QE5œ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "SecretInput-b1QE5",
+ "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b1QE5œ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "AdvancedDynamicFormBuilder-81Exw",
+ "targetHandle": "{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
+ },
+ {
+ "animated": false,
+ "data": {
+ "sourceHandle": {
+ "dataType": "SecretInput",
+ "id": "SecretInput-KxF3a",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "dynamic_source_url",
+ "id": "AdvancedDynamicFormBuilder-81Exw",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__SecretInput-KxF3a{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-KxF3aœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "SecretInput-KxF3a",
+ "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-KxF3aœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "AdvancedDynamicFormBuilder-81Exw",
+ "targetHandle": "{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
}
],
"nodes": [
@@ -625,6 +681,7 @@
"tool_mode": false
},
"selected_output": "chunks",
+ "showNode": false,
"type": "SplitText"
},
"dragging": false,
@@ -635,8 +692,8 @@
"width": 320
},
"position": {
- "x": 1711.934915237861,
- "y": 1637.2034518030887
+ "x": 1354.9418688156868,
+ "y": 1768.0815618919698
},
"positionAbsolute": {
"x": 1683.4543896546102,
@@ -667,7 +724,7 @@
],
"frozen": false,
"icon": "braces",
- "last_updated": "2025-12-12T20:12:18.129Z",
+ "last_updated": "2026-02-06T18:04:39.034Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {},
@@ -717,7 +774,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"code": {
@@ -771,6 +828,39 @@
"type": "str",
"value": ""
},
+ "dynamic_document_id": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "document_id",
+ "dynamic": false,
+ "helper_text": null,
+ "info": " (Can connect to: Text, Message)",
+ "input_types": [
+ "Text",
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": false,
+ "multiline": true,
+ "name": "dynamic_document_id",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": null,
+ "refresh_button": null,
+ "refresh_button_text": null,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": ""
+ },
"dynamic_owner": {
"_input_type": "MultilineInput",
"advanced": false,
@@ -870,6 +960,39 @@
"type": "str",
"value": ""
},
+ "dynamic_source_url": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "source_url",
+ "dynamic": false,
+ "helper_text": null,
+ "info": " (Can connect to: Text, Message)",
+ "input_types": [
+ "Text",
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": false,
+ "multiline": true,
+ "name": "dynamic_source_url",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": null,
+ "refresh_button": null,
+ "refresh_button_text": null,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": ""
+ },
"form_fields": {
"_input_type": "TableInput",
"advanced": false,
@@ -944,6 +1067,14 @@
{
"field_name": "connector_type",
"field_type": "Text"
+ },
+ {
+ "field_name": "source_url",
+ "field_type": "Text"
+ },
+ {
+ "field_name": "document_id",
+ "field_type": "Text"
}
]
},
@@ -970,18 +1101,18 @@
"tool_mode": false
},
"selected_output": "form_data",
- "showNode": true,
+ "showNode": false,
"type": "AdvancedDynamicFormBuilder"
},
"dragging": false,
"id": "AdvancedDynamicFormBuilder-81Exw",
"measured": {
- "height": 552,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1778.514096901592,
- "y": 673.5870187063417
+ "x": 1352.0744672643646,
+ "y": 1508.919401750829
},
"selected": false,
"type": "genericNode"
@@ -1070,18 +1201,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "SecretInput"
},
"dragging": false,
"id": "SecretInput-F34VJ",
"measured": {
- "height": 220,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1343.2266447254406,
- "y": 503.74766485111434
+ "x": 1089.1452135742702,
+ "y": 1355.8795635459912
},
"selected": false,
"type": "genericNode"
@@ -1170,18 +1301,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "SecretInput"
},
"dragging": false,
"id": "SecretInput-b2cab",
"measured": {
- "height": 220,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1341.4694747797632,
- "y": 753.9209691638071
+ "x": 1085.2410189000832,
+ "y": 1484.8702711989886
},
"selected": false,
"type": "genericNode"
@@ -1270,18 +1401,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "SecretInput"
},
"dragging": false,
"id": "SecretInput-ZVfuS",
"measured": {
- "height": 220,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1336.2669044250051,
- "y": 1000.9543699805594
+ "x": 1083.5090209646805,
+ "y": 1549.912686959435
},
"selected": false,
"type": "genericNode"
@@ -1370,18 +1501,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "SecretInput"
},
"dragging": false,
"id": "SecretInput-Iqtxd",
"measured": {
- "height": 220,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1342.0034534756019,
- "y": 1239.4741232342342
+ "x": 1084.150007027148,
+ "y": 1616.4708779408236
},
"selected": false,
"type": "genericNode"
@@ -1496,7 +1627,7 @@
"info": "URL of the Docling Serve instance.",
"list": false,
"list_add_label": "Add More",
- "load_from_db": true,
+ "load_from_db": false,
"name": "api_url",
"override_skip": false,
"placeholder": "",
@@ -1507,7 +1638,7 @@
"trace_as_metadata": true,
"track_in_telemetry": false,
"type": "str",
- "value": "DOCLING_SERVE_URL"
+ "value": "http://localhost:5001"
},
"code": {
"advanced": true,
@@ -1785,18 +1916,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "DoclingRemote"
},
"dragging": false,
"id": "DoclingRemote-Dp3PX",
"measured": {
- "height": 312,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": -175.49741984154275,
- "y": 1505.0245107748437
+ "x": -27.54468027620713,
+ "y": 1770.6264167415306
},
"selected": false,
"type": "genericNode"
@@ -2042,18 +2173,18 @@
"tool_mode": false
},
"selected_output": "dataframe",
- "showNode": true,
+ "showNode": false,
"type": "ExportDoclingDocument"
},
"dragging": false,
"id": "ExportDoclingDocument-zZdRg",
"measured": {
- "height": 347,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 206.97755086947473,
- "y": 1610.6498167995744
+ "x": 258.9758629312913,
+ "y": 1768.6446880643248
},
"selected": false,
"type": "genericNode"
@@ -2090,7 +2221,7 @@
],
"frozen": false,
"icon": "table",
- "last_updated": "2025-12-12T20:12:18.208Z",
+ "last_updated": "2026-02-06T17:43:52.579Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -2137,7 +2268,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"ascending": {
@@ -2493,18 +2624,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "DataFrameOperations"
},
"dragging": false,
"id": "DataFrameOperations-1BWXB",
"measured": {
- "height": 399,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 575.218683966709,
- "y": 1607.3264908868193
+ "x": 533.2200473013957,
+ "y": 1769.3212323101711
},
"selected": false,
"type": "genericNode"
@@ -2541,7 +2672,7 @@
],
"frozen": false,
"icon": "table",
- "last_updated": "2025-12-12T20:12:18.209Z",
+ "last_updated": "2026-02-06T17:43:52.579Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -2588,7 +2719,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"ascending": {
@@ -2944,18 +3075,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "DataFrameOperations"
},
"dragging": false,
"id": "DataFrameOperations-N80fC",
"measured": {
- "height": 399,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1330.149865256777,
- "y": 1619.9268393528012
+ "x": 1070.158304947694,
+ "y": 1767.9220352210484
},
"selected": false,
"type": "genericNode"
@@ -2992,7 +3123,7 @@
],
"frozen": false,
"icon": "table",
- "last_updated": "2025-12-12T20:12:18.209Z",
+ "last_updated": "2026-02-06T17:43:52.580Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -3039,7 +3170,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"ascending": {
@@ -3395,18 +3526,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "DataFrameOperations"
},
"dragging": false,
"id": "DataFrameOperations-9vMrp",
"measured": {
- "height": 399,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 937.1310281139399,
- "y": 1611.2186890450444
+ "x": 787.135897166392,
+ "y": 1770.3912596804048
},
"selected": false,
"type": "genericNode"
@@ -3430,8 +3561,8 @@
"width": 1000
},
"position": {
- "x": -184.83853691310878,
- "y": 944.7352701051674
+ "x": -21.449808371464087,
+ "y": 1292.5429595750052
},
"resizing": true,
"selected": false,
@@ -3950,7 +4081,7 @@
"info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).",
"list": false,
"list_add_label": "Add More",
- "load_from_db": true,
+ "load_from_db": false,
"name": "opensearch_url",
"override_skip": false,
"placeholder": "",
@@ -3961,7 +4092,7 @@
"trace_as_metadata": true,
"track_in_telemetry": false,
"type": "str",
- "value": "OPENSEARCH_URL"
+ "value": "https://opensearch:9200"
},
"password": {
"_input_type": "SecretStrInput",
@@ -4143,18 +4274,18 @@
"tool_mode": false
},
"selected_output": "search_results",
- "showNode": true,
+ "showNode": false,
"type": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding"
},
"dragging": false,
"id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
"measured": {
- "height": 904,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 2261.865622928042,
- "y": 1349.2821108833643
+ "x": 1906.1233004091807,
+ "y": 1854.130454249347
},
"selected": false,
"type": "genericNode"
@@ -4193,7 +4324,7 @@
],
"frozen": false,
"icon": "binary",
- "last_updated": "2025-12-12T20:12:18.131Z",
+ "last_updated": "2026-02-06T18:05:46.249Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -4261,13 +4392,13 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"api_base": {
"_input_type": "MessageTextInput",
- "advanced": true,
- "display_name": "API Base URL",
+ "advanced": false,
+ "display_name": "OpenAI API Base URL",
"dynamic": false,
"info": "Base URL for the API. Leave empty for default.",
"input_types": [
@@ -4280,7 +4411,7 @@
"override_skip": false,
"placeholder": "",
"required": false,
- "show": false,
+ "show": true,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
@@ -4292,7 +4423,7 @@
"api_key": {
"_input_type": "SecretStrInput",
"advanced": false,
- "display_name": "IBM watsonx.ai API Key",
+ "display_name": "OpenAI API Key",
"dynamic": false,
"info": "Model Provider API key",
"input_types": [],
@@ -4307,7 +4438,7 @@
"title_case": false,
"track_in_telemetry": false,
"type": "str",
- "value": "WATSONX_API_KEY"
+ "value": "OPENAI_API_KEY"
},
"base_url_ibm_watsonx": {
"_input_type": "DropdownInput",
@@ -4332,7 +4463,7 @@
"placeholder": "",
"real_time_refresh": true,
"required": false,
- "show": true,
+ "show": false,
"title_case": false,
"toggle": false,
"tool_mode": false,
@@ -4433,7 +4564,7 @@
"override_skip": false,
"placeholder": "",
"required": false,
- "show": true,
+ "show": false,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
@@ -4474,7 +4605,7 @@
"load_from_db": false,
"name": "model",
"options": [
- "ibm/granite-embedding-278m-multilingual"
+ "text-embedding-3-small"
],
"options_metadata": [],
"override_skip": false,
@@ -4489,7 +4620,7 @@
"trace_as_metadata": true,
"track_in_telemetry": true,
"type": "str",
- "value": "ibm/granite-embedding-278m-multilingual"
+ "value": "text-embedding-3-small"
},
"model_kwargs": {
"_input_type": "DictInput",
@@ -4553,7 +4684,7 @@
"override_skip": false,
"placeholder": "",
"required": false,
- "show": true,
+ "show": false,
"title_case": false,
"tool_mode": false,
"trace_as_input": true,
@@ -4603,7 +4734,7 @@
"trace_as_metadata": true,
"track_in_telemetry": true,
"type": "str",
- "value": "IBM watsonx.ai"
+ "value": "OpenAI"
},
"request_timeout": {
"_input_type": "FloatInput",
@@ -4657,7 +4788,7 @@
"override_skip": false,
"placeholder": "",
"required": false,
- "show": true,
+ "show": false,
"title_case": false,
"tool_mode": false,
"trace_as_metadata": true,
@@ -4668,18 +4799,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "EmbeddingModel"
},
"dragging": false,
"id": "EmbeddingModel-EAo9i",
"measured": {
- "height": 534,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 838.9563350647003,
- "y": 2191.523005861695
+ "x": 1355.5927560779853,
+ "y": 1905.3991707633513
},
"selected": false,
"type": "genericNode"
@@ -4718,7 +4849,7 @@
],
"frozen": false,
"icon": "binary",
- "last_updated": "2025-12-12T20:12:18.132Z",
+ "last_updated": "2026-02-06T18:05:43.252Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -4786,7 +4917,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"api_base": {
@@ -5194,18 +5325,18 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "EmbeddingModel"
},
"dragging": false,
"id": "EmbeddingModel-E0hvR",
"measured": {
- "height": 369,
- "width": 320
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1223.4804629271505,
- "y": 2198.8989246514284
+ "x": 1361.1152821119326,
+ "y": 1983.3161387480468
},
"selected": false,
"type": "genericNode"
@@ -5244,7 +5375,7 @@
],
"frozen": false,
"icon": "binary",
- "last_updated": "2025-12-12T20:12:18.133Z",
+ "last_updated": "2026-02-06T18:05:39.873Z",
"legacy": false,
"lf_version": "1.7.0.dev21",
"metadata": {
@@ -5312,7 +5443,7 @@
"value": "5488df7c-b93f-4f87-a446-b67028bc0813"
},
"_frontend_node_folder_id": {
- "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3"
+ "value": "c557e756-bc37-48fd-972d-366ef037c8cf"
},
"_type": "Component",
"api_base": {
@@ -5719,27 +5850,227 @@
},
"tool_mode": false
},
- "showNode": true,
+ "showNode": false,
"type": "EmbeddingModel"
},
"dragging": false,
"id": "EmbeddingModel-3LsIP",
"measured": {
- "height": 369,
- "width": 320
+ "height": 48,
+ "width": 192
+ },
+ "position": {
+ "x": 1360.2396840279503,
+ "y": 2067.099324740414
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "SecretInput-b1QE5",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Allows the selection of a secret to be generated as output..",
+ "display_name": "Secret Input",
+ "documentation": "https://docs.langflow.org/components-io#text-input",
+ "edited": true,
+ "field_order": [
+ "input_value"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "legacy": false,
+ "lf_version": "1.7.0.dev21",
+ "metadata": {},
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "hidden": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n"
+ },
+ "input_value": {
+ "_input_type": "SecretStrInput",
+ "advanced": false,
+ "display_name": "Secret",
+ "dynamic": false,
+ "info": "Secret to be passed as input.",
+ "input_types": [],
+ "load_from_db": true,
+ "name": "input_value",
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "type": "str",
+ "value": "DOCUMENT_ID"
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "SecretInput"
+ },
+ "dragging": false,
+ "id": "SecretInput-b1QE5",
+ "measured": {
+ "height": 48,
+ "width": 192
+ },
+ "position": {
+ "x": 1085.4357429636027,
+ "y": 1419.65527860865
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "SecretInput-KxF3a",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Allows the selection of a secret to be generated as output..",
+ "display_name": "Secret Input",
+ "documentation": "https://docs.langflow.org/components-io#text-input",
+ "edited": true,
+ "field_order": [
+ "input_value"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "legacy": false,
+ "lf_version": "1.7.0.dev21",
+ "metadata": {},
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "hidden": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n"
+ },
+ "input_value": {
+ "_input_type": "SecretStrInput",
+ "advanced": false,
+ "display_name": "Secret",
+ "dynamic": false,
+ "info": "Secret to be passed as input.",
+ "input_types": [],
+ "load_from_db": true,
+ "name": "input_value",
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "type": "str",
+ "value": "SOURCE_URL"
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "SecretInput"
+ },
+ "dragging": false,
+ "id": "SecretInput-KxF3a",
+ "measured": {
+ "height": 48,
+ "width": 192
},
"position": {
- "x": 1638.9179466145608,
- "y": 2110.0422159522327
+ "x": 1085.9081887151694,
+ "y": 1680.6730861389885
},
"selected": false,
"type": "genericNode"
}
],
"viewport": {
- "x": 249.3666737262397,
- "y": -156.8776378758762,
- "zoom": 0.38977017930844676
+ "x": 47.599403481811805,
+ "y": -629.4207733468738,
+ "zoom": 0.6293990172646186
}
},
"description": "Load your data for chat context with Retrieval Augmented Generation.",
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.ts
new file mode 100644
index 000000000..e87f92a17
--- /dev/null
+++ b/frontend/app/api/mutations/useSyncConnector.ts
@@ -0,0 +1,77 @@
+"use client";
+
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+// Response types
+interface SyncResponse {
+ task_ids?: string[];
+ status: string;
+ message: string;
+ connections_synced?: number;
+ synced_connectors?: string[];
+ skipped_connectors?: string[];
+ errors?: Array<{ connector_type: string; error: string }> | null;
+}
+
+// Sync all cloud connectors
+const syncAllConnectors = async (): Promise<SyncResponse> => {
+ const response = await fetch("/api/connectors/sync-all", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+
+ if (!response.ok) {
+ const error = await response.json();
+ throw new Error(error.error || "Failed to sync connectors");
+ }
+
+ return response.json();
+};
+
+// Sync a specific connector type
+const syncConnector = async (connectorType: string): Promise<SyncResponse> => {
+ const response = await fetch(`/api/connectors/${connectorType}/sync`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({}),
+ });
+
+ if (!response.ok) {
+ const error = await response.json();
+ throw new Error(error.error || `Failed to sync ${connectorType}`);
+ }
+
+ return response.json();
+};
+
+export const useSyncAllConnectors = () => {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: syncAllConnectors,
+ onSettled: () => {
+ // Invalidate and refetch search queries after a delay to allow sync to complete
+ setTimeout(() => {
+ queryClient.invalidateQueries({ queryKey: ["search"] });
+ }, 2000);
+ },
+ });
+};
+
+export const useSyncConnector = () => {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: syncConnector,
+ onSettled: () => {
+ // Invalidate and refetch search queries after a delay to allow sync to complete
+ setTimeout(() => {
+ queryClient.invalidateQueries({ queryKey: ["search"] });
+ }, 2000);
+ },
+ });
+};
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 85189a85f..0277a3344 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -8,7 +8,7 @@ import {
type ValueFormatterParams,
} from "ag-grid-community";
import { AgGridReact, type CustomCellRendererProps } from "ag-grid-react";
-import { Cloud, FileIcon, Globe } from "lucide-react";
+import { Cloud, FileIcon, Globe, RefreshCw } from "lucide-react";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useRef, useState } from "react";
import { KnowledgeDropdown } from "@/components/knowledge-dropdown";
@@ -44,6 +44,7 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";
import OneDriveIcon from "../../components/icons/one-drive-logo";
import SharePointIcon from "../../components/icons/share-point-logo";
import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
+import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
// Function to get the appropriate icon for a connector type
function getSourceIcon(connectorType?: string) {
@@ -78,6 +79,7 @@ function SearchPage() {
const lastErrorRef = useRef<string | null>(null);
const deleteDocumentMutation = useDeleteDocument();
+ const syncAllConnectorsMutation = useSyncAllConnectors();
useEffect(() => {
refreshTasks();
@@ -297,7 +299,12 @@ function SearchPage() {
if (status !== "active") {
return null;
}
- return <KnowledgeActionsDropdown filename={data?.filename || ""} />;
+ return (
+ <KnowledgeActionsDropdown
+ filename={data?.filename || ""}
+ connectorType={data?.connector_type}
+ />
+ );
},
cellStyle: {
alignItems: "center",
@@ -380,15 +387,43 @@ function SearchPage() {
{/* Search Input Area */}
<div className="flex-1 flex items-center flex-shrink-0 flex-wrap-reverse gap-3 mb-6">
<KnowledgeSearchInput />
- {/* //TODO: Implement sync button */}
- {/* <Button
- type="button"
- variant="outline"
- className="rounded-lg flex-shrink-0"
- onClick={() => alert("Not implemented")}
- >
- Sync
- </Button> */}
+ <Button
+ type="button"
+ variant="outline"
+ className="rounded-lg flex-shrink-0"
+ disabled={syncAllConnectorsMutation.isPending}
+ onClick={async () => {
+ try {
+ toast.info("Syncing all cloud connectors...");
+ const result = await syncAllConnectorsMutation.mutateAsync();
+ if (result.status === "no_files") {
+ toast.info(result.message || "No cloud files to sync. Add files from cloud connectors first.");
+ } else if (result.synced_connectors && result.synced_connectors.length > 0) {
+ toast.success(
+ `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`
+ );
+ } else if (result.errors && result.errors.length > 0) {
+ toast.error("Some connectors failed to sync");
+ }
+ } catch (error) {
+ toast.error(
+ error instanceof Error ? error.message : "Failed to sync connectors"
+ );
+ }
+ }}
+ >
+ {syncAllConnectorsMutation.isPending ? (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />
+ Syncing...
+ </>
+ ) : (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2" />
+ Sync
+ </>
+ )}
+ </Button>
{selectedRows.length > 0 && (
<Button
type="button"
diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsx
index a144c8ac3..45eae1e36 100644
--- a/frontend/components/knowledge-actions-dropdown.tsx
+++ b/frontend/components/knowledge-actions-dropdown.tsx
@@ -1,10 +1,11 @@
"use client";
-import { EllipsisVertical } from "lucide-react";
+import { EllipsisVertical, RefreshCw } from "lucide-react";
import { useRouter } from "next/navigation";
import { useState } from "react";
import { toast } from "sonner";
import { useDeleteDocument } from "@/app/api/mutations/useDeleteDocument";
+import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import {
DropdownMenu,
DropdownMenuContent,
@@ -16,15 +17,24 @@ import { Button } from "./ui/button";
interface KnowledgeActionsDropdownProps {
filename: string;
+ connectorType?: string;
}
+// Cloud connector types that support sync
+const CLOUD_CONNECTOR_TYPES = new Set(["google_drive", "onedrive", "sharepoint"]);
+
export const KnowledgeActionsDropdown = ({
filename,
+ connectorType,
}: KnowledgeActionsDropdownProps) => {
const [showDeleteDialog, setShowDeleteDialog] = useState(false);
const deleteDocumentMutation = useDeleteDocument();
+ const syncConnectorMutation = useSyncConnector();
const router = useRouter();
+ // Check if this file is from a cloud connector (can be synced)
+ const isCloudFile = connectorType && CLOUD_CONNECTOR_TYPES.has(connectorType);
+
const handleDelete = async () => {
try {
await deleteDocumentMutation.mutateAsync({ filename });
@@ -37,6 +47,26 @@ export const KnowledgeActionsDropdown = ({
}
};
+ const handleSync = async () => {
+ if (!connectorType) return;
+
+ try {
+ toast.info(`Syncing ${connectorType}...`);
+ const result = await syncConnectorMutation.mutateAsync(connectorType);
+ if (result.status === "no_files") {
+ toast.info(result.message || `No ${connectorType} files to sync.`);
+ } else if (result.task_ids && result.task_ids.length > 0) {
+ toast.success(
+ `Sync started for ${connectorType}. Check task notifications for progress.`
+ );
+ }
+ } catch (error) {
+ toast.error(
+ error instanceof Error ? error.message : `Failed to sync ${connectorType}`,
+ );
+ }
+ };
+
return (
<>
<DropdownMenu>
@@ -56,19 +86,25 @@ export const KnowledgeActionsDropdown = ({
>
View chunks
</DropdownMenuItem>
- {/* //TODO: Implement rename and sync */}
- {/* <DropdownMenuItem
- className="text-primary focus:text-primary"
- onClick={() => alert("Not implemented")}
- >
- Rename
- </DropdownMenuItem>
- <DropdownMenuItem
- className="text-primary focus:text-primary"
- onClick={() => alert("Not implemented")}
- >
- Sync
- </DropdownMenuItem> */}
+ {isCloudFile && (
+ <DropdownMenuItem
+ className="text-primary focus:text-primary"
+ disabled={syncConnectorMutation.isPending}
+ onClick={handleSync}
+ >
+ {syncConnectorMutation.isPending ? (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2 animate-spin" />
+ Syncing...
+ </>
+ ) : (
+ <>
+ <RefreshCw className="h-4 w-4 mr-2" />
+ Sync
+ </>
+ )}
+ </DropdownMenuItem>
+ )}
<DropdownMenuItem
className="text-destructive focus:text-destructive"
onClick={() => setShowDeleteDialog(true)}
diff --git a/helm/openrag/templates/langflow/deployment.yaml b/helm/openrag/templates/langflow/deployment.yaml
index 2683011f4..377235401 100644
--- a/helm/openrag/templates/langflow/deployment.yaml
+++ b/helm/openrag/templates/langflow/deployment.yaml
@@ -100,6 +100,10 @@ spec:
value: "system"
- name: CONNECTOR_TYPE_URL
value: "url"
+ - name: DOCUMENT_ID
+ value: ""
+ - name: SOURCE_URL
+ value: ""
- name: OPENRAG-QUERY-FILTER
value: "{}"
- name: FILENAME
diff --git a/helm/openrag/values.yaml b/helm/openrag/values.yaml
index fbf0707a3..2a2b52371 100644
--- a/helm/openrag/values.yaml
+++ b/helm/openrag/values.yaml
@@ -92,7 +92,7 @@ langflow:
logLevel: "INFO" # DEBUG, INFO, WARNING, ERROR
# Variables to expose to flows
- variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL"
+ variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL"
# Probes
livenessProbe:
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 9824e6a71..4c54097c7 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -2,10 +2,82 @@ from starlette.requests import Request
from starlette.responses import JSONResponse, PlainTextResponse
from utils.logging_config import get_logger
from utils.telemetry import TelemetryClient, Category, MessageId
+from config.settings import INDEX_NAME
logger = get_logger(__name__)
+async def get_synced_file_ids_for_connector(
+ connector_type: str,
+ user_id: str,
+ session_manager,
+ jwt_token: str = None,
+) -> tuple:
+ """
+ Query OpenSearch for unique document_id values where connector_type matches.
+ Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.
+
+ Note: Langflow-ingested files may not have document_id stored. In that case,
+ filenames are returned for filename-based filtering during sync.
+ """
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ # Query for both document_id and filename aggregations
+ query_body = {
+ "size": 0,
+ "query": {
+ "term": {
+ "connector_type": connector_type
+ }
+ },
+ "aggs": {
+ "unique_document_ids": {
+ "terms": {
+ "field": "document_id",
+ "size": 10000
+ }
+ },
+ "unique_filenames": {
+ "terms": {
+ "field": "filename",
+ "size": 10000
+ }
+ }
+ }
+ }
+
+ result = await opensearch_client.search(
+ index=INDEX_NAME,
+ body=query_body
+ )
+
+ # Get document_ids (preferred - these are the actual connector file IDs)
+ doc_id_buckets = result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
+ file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]
+
+ # Get filenames as fallback
+ filename_buckets = result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
+ filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]
+
+ logger.debug(
+ "Found synced files for connector",
+ connector_type=connector_type,
+ file_ids_count=len(file_ids),
+ filenames_count=len(filenames),
+ )
+
+ return file_ids, filenames
+
+ except Exception as e:
+ logger.error(
+ "Failed to get synced file IDs",
+ connector_type=connector_type,
+ error=str(e),
+ )
+ return [], []
+
+
async def list_connectors(request: Request, connector_service, session_manager):
"""List available connector types with metadata"""
try:
@@ -104,6 +176,7 @@ async def connector_sync(request: Request, connector_service, session_manager):
)
if selected_files:
+ # Explicit files selected (e.g., from file picker) - sync those specific files
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
@@ -112,12 +185,52 @@ async def connector_sync(request: Request, connector_service, session_manager):
file_infos=file_infos,
)
else:
- task_id = await connector_service.sync_connector_files(
- working_connection.connection_id,
- user.user_id,
- max_files,
+ # No files specified - sync only files already in OpenSearch for this connector
+ # This ensures deleted files stay deleted
+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
jwt_token=jwt_token,
)
+
+ if not existing_file_ids and not existing_filenames:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": f"No {connector_type} files to sync. Add files from the connector first.",
+ },
+ status_code=200,
+ )
+
+ # If we have document_ids (connector file IDs), use sync_specific_files
+ # Otherwise, use filename filtering with sync_connector_files
+ if existing_file_ids:
+ logger.info(
+ "Syncing specific files by document_id",
+ connector_type=connector_type,
+ file_count=len(existing_file_ids),
+ )
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ existing_file_ids,
+ jwt_token=jwt_token,
+ )
+ else:
+ # Fallback: use filename filtering (for Langflow-ingested files without document_id)
+ logger.info(
+ "Syncing files by filename filter (document_id not available)",
+ connector_type=connector_type,
+ filename_count=len(existing_filenames),
+ )
+ task_id = await connector_service.sync_connector_files(
+ working_connection.connection_id,
+ user.user_id,
+ max_files=None,
+ jwt_token=jwt_token,
+ filename_filter=set(existing_filenames),
+ )
task_ids = [task_id]
await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)
return JSONResponse(
@@ -467,6 +580,159 @@ async def connector_disconnect(request: Request, connector_service, session_mana
)
+async def sync_all_connectors(request: Request, connector_service, session_manager):
+ """
+ Sync files from all active cloud connector connections (Google Drive, OneDrive, SharePoint).
+
+ Only syncs files that are already indexed in OpenSearch - this ensures deleted files
+ stay deleted and only previously selected files get re-synced (to update content and ACLs).
+ """
+ try:
+ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_START)
+ user = request.state.user
+ jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token)
+
+ # Cloud connector types to sync
+ cloud_connector_types = ["google_drive", "onedrive", "sharepoint"]
+
+ all_task_ids = []
+ synced_connectors = []
+ skipped_connectors = []
+ errors = []
+
+ for connector_type in cloud_connector_types:
+ try:
+ # First, get existing file IDs/filenames from OpenSearch for this connector type
+ existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+
+ if not existing_file_ids and not existing_filenames:
+ logger.debug(
+ "No existing files in OpenSearch for connector type, skipping",
+ connector_type=connector_type,
+ )
+ skipped_connectors.append(connector_type)
+ continue
+
+ # Get all active connections for this connector type and user
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type=connector_type
+ )
+
+ active_connections = [conn for conn in connections if conn.is_active]
+ if not active_connections:
+ logger.debug(
+ "No active connections for connector type",
+ connector_type=connector_type,
+ )
+ continue
+
+ # Find the first connection that actually works
+ working_connection = None
+ for connection in active_connections:
+ try:
+ connector = await connector_service.get_connector(connection.connection_id)
+ if connector and await connector.authenticate():
+ working_connection = connection
+ break
+ except Exception as e:
+ logger.debug(
+ "Connection validation failed",
+ connection_id=connection.connection_id,
+ error=str(e),
+ )
+ continue
+
+ if not working_connection:
+ logger.debug(
+ "No working connection for connector type",
+ connector_type=connector_type,
+ )
+ continue
+
+ # Sync using document_ids if available, else use filename filter
+ if existing_file_ids:
+ logger.info(
+ "Syncing specific files by document_id",
+ connector_type=connector_type,
+ file_count=len(existing_file_ids),
+ )
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ existing_file_ids,
+ jwt_token=jwt_token,
+ )
+ else:
+ # Fallback: use filename filtering
+ logger.info(
+ "Syncing files by filename filter",
+ connector_type=connector_type,
+ filename_count=len(existing_filenames),
+ )
+ task_id = await connector_service.sync_connector_files(
+ working_connection.connection_id,
+ user.user_id,
+ max_files=None,
+ jwt_token=jwt_token,
+ filename_filter=set(existing_filenames),
+ )
+
+ all_task_ids.append(task_id)
+ synced_connectors.append(connector_type)
+ logger.info(
+ "Started sync for connector type",
+ connector_type=connector_type,
+ task_id=task_id,
+ file_count=len(existing_file_ids) if existing_file_ids else len(existing_filenames),
+ )
+
+ except Exception as e:
+ logger.error(
+ "Failed to sync connector type",
+ connector_type=connector_type,
+ error=str(e),
+ )
+ errors.append({"connector_type": connector_type, "error": str(e)})
+
+ if not all_task_ids and not errors:
+ if skipped_connectors:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": "No files to sync. Add files from cloud connectors first.",
+ "skipped_connectors": skipped_connectors,
+ },
+ status_code=200,
+ )
+ return JSONResponse(
+ {"error": "No active cloud connector connections found"},
+ status_code=404,
+ )
+
+ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)
+ return JSONResponse(
+ {
+ "task_ids": all_task_ids,
+ "status": "sync_started",
+ "message": f"Started syncing files from {len(synced_connectors)} cloud connector(s)",
+ "synced_connectors": synced_connectors,
+ "skipped_connectors": skipped_connectors if skipped_connectors else None,
+ "errors": errors if errors else None,
+ },
+ status_code=201,
+ )
+
+ except Exception as e:
+ logger.error("Sync all connectors failed", error=str(e))
+ await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_FAILED)
+ return JSONResponse({"error": f"Sync failed: {str(e)}"}, status_code=500)
+
+
async def connector_token(request: Request, connector_service, session_manager):
"""Get access token for connector API calls (e.g., Pickers)."""
url_connector_type = request.path_params.get("connector_type")
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 49be8588b..3d5eeda5d 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -64,12 +64,26 @@ class LangflowConnectorService:
# Step 1: Upload file to Langflow
logger.debug("Uploading file to Langflow", filename=document.filename)
content = document.content
+ processed_filename = document.filename.replace(" ", "_").replace("/", "_") + suffix
file_tuple = (
- document.filename.replace(" ", "_").replace("/", "_") + suffix,
+ processed_filename,
content,
document.mimetype or "application/octet-stream",
)
+ # Step 0: Delete existing chunks for this file before re-ingesting
+ # This prevents duplicate chunks when syncing files
+ if self.session_manager:
+ try:
+ from config.settings import INDEX_NAME
+ opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token)
+ delete_body = {"query": {"term": {"filename": processed_filename}}}
+ delete_result = await opensearch_client.delete_by_query(index=INDEX_NAME, body=delete_body)
+ deleted_count = delete_result.get("deleted", 0)
+ logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count)
+ except Exception as delete_err:
+ logger.warning("Failed to delete existing chunks before re-ingestion", filename=processed_filename, error=str(delete_err))
+
langflow_file_id = None # Initialize to track if upload succeeded
try:
upload_result = await self.langflow_service.upload_user_file(
@@ -101,6 +115,8 @@ class LangflowConnectorService:
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
+ document_id=document.id,
+ source_url=document.source_url,
)
logger.debug("Ingestion flow completed", result=ingestion_result)
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 64c74f5cf..2bc6e9986 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -200,8 +200,20 @@ class ConnectorService:
user_id: str,
max_files: int = None,
jwt_token: str = None,
+ filename_filter: set = None,
) -> str:
- """Sync files from a connector connection using existing task tracking system"""
+ """
+ Sync files from a connector connection using existing task tracking system.
+
+ Args:
+ connection_id: The connection ID
+ user_id: The user ID
+ max_files: Maximum number of files to sync
+ jwt_token: Optional JWT token
+ filename_filter: Optional set of filenames to filter - only files with names
+ in this set will be synced. Used to prevent deleted files
+ from being re-synced.
+ """
if not self.task_service:
raise ValueError(
"TaskService not available - connector sync requires task service dependency"
@@ -248,6 +260,15 @@ class ConnectorService:
for file_info in files:
if max_files and len(files_to_process) >= max_files:
break
+ # Filter by filename if filter is provided
+ if filename_filter is not None:
+ file_name = file_info.get("name", "")
+ if file_name not in filename_filter:
+ logger.debug(
+ "Skipping file not in filter",
+ filename=file_name,
+ )
+ continue
files_to_process.append(file_info)
# Stop if we have enough files or no more pages
diff --git a/src/main.py b/src/main.py
index 6c8b696b2..60903746f 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1077,6 +1077,17 @@ async def create_app():
),
methods=["POST"],
),
+ Route(
+ "/connectors/sync-all",
+ require_auth(services["session_manager"])(
+ partial(
+ connectors.sync_all_connectors,
+ connector_service=services["connector_service"],
+ session_manager=services["session_manager"],
+ )
+ ),
+ methods=["POST"],
+ ),
Route(
"/connectors/{connector_type}/status",
require_auth(services["session_manager"])(
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index ca5fec7ca..e670e2b26 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -68,6 +68,8 @@ class LangflowFileService:
owner_name: Optional[str] = None,
owner_email: Optional[str] = None,
connector_type: Optional[str] = None,
+ document_id: Optional[str] = None,
+ source_url: Optional[str] = None,
) -> Dict[str, Any]:
"""
Trigger the ingestion flow with provided file paths.
@@ -157,6 +159,8 @@ class LangflowFileService:
"X-Langflow-Global-Var-MIMETYPE": mimetype,
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
+ "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id) if document_id else "",
+ "X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "",
}
# Add provider credentials as global variables for ingestion