Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: add replace_duplicates option for Google Drive sync (#1674)

Rico Furtado0961fd1bc8bc
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 309158337..bd5c7dac7 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -19,6 +19,11 @@ from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
+def _connector_sync_should_replace(connector_type: str) -> bool:
+ """Return True for connector types where sync should replace existing indexed files."""
+ return connector_type in ["google_drive", "sharepoint", "onedrive"]
+
+
async def get_synced_file_ids_for_connector(
connector_type: str,
user_id: str,
@@ -508,6 +513,7 @@ async def connector_sync(
user.user_id,
existing_file_ids,
jwt_token=jwt_token,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
)
else:
# Fallback: use filename filtering (for Langflow-ingested files without document_id)
@@ -522,6 +528,7 @@ async def connector_sync(
max_files=None,
jwt_token=jwt_token,
filename_filter=set(existing_filenames),
+ replace_duplicates=_connector_sync_should_replace(connector_type),
)
task_ids = [task_id]
await TelemetryClient.send_event(
@@ -979,6 +986,7 @@ async def sync_all_connectors(
user.user_id,
existing_file_ids,
jwt_token=jwt_token,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
)
else:
# Fallback: use filename filtering
@@ -993,6 +1001,7 @@ async def sync_all_connectors(
max_files=None,
jwt_token=jwt_token,
filename_filter=set(existing_filenames),
+ replace_duplicates=_connector_sync_should_replace(connector_type),
)
all_task_ids.append(task_id)
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 6b5b9e01c..e7ee81cd2 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -229,6 +229,8 @@ class LangflowConnectorService:
user_id: str,
max_files: int = None,
jwt_token: str = None,
+ filename_filter: set = None,
+ replace_duplicates: bool = False,
) -> str:
"""Sync files from a connector connection using Langflow processing"""
if not self.task_service:
@@ -271,6 +273,11 @@ class LangflowConnectorService:
for file_info in files:
if max_files and len(files_to_process) >= max_files:
break
+ if filename_filter is not None:
+ file_name = file_info.get("name", "")
+ if file_name not in filename_filter:
+ logger.debug("Skipping file not in filter", filename=file_name)
+ continue
files_to_process.append(file_info)
# Stop if we have enough files or no more pages
@@ -294,6 +301,7 @@ class LangflowConnectorService:
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
+ replace_duplicates=replace_duplicates,
)
# Use file IDs as items
diff --git a/src/connectors/service.py b/src/connectors/service.py
index cfdba9735..9e887a677 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -202,6 +202,7 @@ class ConnectorService:
max_files: int = None,
jwt_token: str = None,
filename_filter: set = None,
+ replace_duplicates: bool = False,
) -> str:
"""
Sync files from a connector connection using existing task tracking system.
@@ -297,6 +298,7 @@ class ConnectorService:
else DocumentService(session_manager=self.session_manager)
),
models_service=self.models_service,
+ replace_duplicates=replace_duplicates,
)
# Use file IDs as items (no more fake file paths!)