OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: add replace_duplicates option for Google Drive sync (#1674)
Rico Furtado0961fd1bc8bc
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 309158337..bd5c7dac7 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -19,6 +19,11 @@ from utils.telemetry import Category, MessageId, TelemetryClientlogger = get_logger(__name__)+def _connector_sync_should_replace(connector_type: str) -> bool:+ """Return True for connector types where sync should replace existing indexed files."""+ return connector_type in ["google_drive", "sharepoint", "onedrive"]++async def get_synced_file_ids_for_connector(connector_type: str,user_id: str,@@ -508,6 +513,7 @@ async def connector_sync(user.user_id,existing_file_ids,jwt_token=jwt_token,+ replace_duplicates=_connector_sync_should_replace(connector_type),)else:# Fallback: use filename filtering (for Langflow-ingested files without document_id)@@ -522,6 +528,7 @@ async def connector_sync(max_files=None,jwt_token=jwt_token,filename_filter=set(existing_filenames),+ replace_duplicates=_connector_sync_should_replace(connector_type),)task_ids = [task_id]await TelemetryClient.send_event(@@ -979,6 +986,7 @@ async def sync_all_connectors(user.user_id,existing_file_ids,jwt_token=jwt_token,+ replace_duplicates=_connector_sync_should_replace(connector_type),)else:# Fallback: use filename filtering@@ -993,6 +1001,7 @@ async def sync_all_connectors(max_files=None,jwt_token=jwt_token,filename_filter=set(existing_filenames),+ replace_duplicates=_connector_sync_should_replace(connector_type),)all_task_ids.append(task_id)diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 6b5b9e01c..e7ee81cd2 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -229,6 +229,8 @@ class LangflowConnectorService:user_id: str,max_files: int = None,jwt_token: str = None,+ filename_filter: set = None,+ replace_duplicates: bool = False,) -> str:"""Sync files from a connector connection using Langflow processing"""if not self.task_service:@@ -271,6 +273,11 @@ class LangflowConnectorService:for file_info in files:if max_files and len(files_to_process) >= max_files:break+ if filename_filter is not None:+ file_name = file_info.get("name", "")+ if file_name not in filename_filter:+ logger.debug("Skipping file not in filter", filename=file_name)+ continuefiles_to_process.append(file_info)# Stop if we have enough files or no more pages@@ -294,6 +301,7 @@ class LangflowConnectorService:jwt_token=jwt_token,owner_name=owner_name,owner_email=owner_email,+ replace_duplicates=replace_duplicates,)# Use file IDs as itemsdiff --git a/src/connectors/service.py b/src/connectors/service.pyindex cfdba9735..9e887a677 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -202,6 +202,7 @@ class ConnectorService:max_files: int = None,jwt_token: str = None,filename_filter: set = None,+ replace_duplicates: bool = False,) -> str:"""Sync files from a connector connection using existing task tracking system.@@ -297,6 +298,7 @@ class ConnectorService:else DocumentService(session_manager=self.session_manager)),models_service=self.models_service,+ replace_duplicates=replace_duplicates,)# Use file IDs as items (no more fake file paths!)