OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: handle orphan files and return appropriate responses when syncing connectors (#1715)
Rico Furtadob13f7d46826e
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsindex 214133429..26566131f 100644--- a/frontend/app/api/mutations/useSyncConnector.ts+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -10,6 +10,7 @@ interface SyncResponse {connections_synced?: number;synced_connectors?: string[];skipped_connectors?: string[];+ deleted_only_connectors?: string[];errors?: Array<{ connector_type: string; error: string }> | null;}diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 4a933b455..665962d87 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -696,8 +696,9 @@ async def connector_sync(# Reconcile orphans (files deleted at the source) before re-syncing.# Strict gating: skip when sync is capped — we'd see a partial remote# listing and delete legitimate files.+ ids_to_sync = list(existing_file_ids)if body.max_files is None:- await reconcile_orphans_for_connector_type(+ orphan_ids = await reconcile_orphans_for_connector_type(connector_type=connector_type,user_id=user.user_id,connector_service=connector_service,@@ -706,10 +707,21 @@ async def connector_sync(existing_file_ids=existing_file_ids,id_field=id_field,)+ if orphan_ids:+ orphan_id_set = set(orphan_ids)+ ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]+ if not ids_to_sync:+ return JSONResponse(+ {+ "status": "no_files",+ "message": f"Deleted stale {connector_type} files; no remaining files to sync.",+ },+ status_code=200,+ )task_id = await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,- existing_file_ids,+ ids_to_sync,jwt_token=jwt_token,replace_duplicates=_connector_sync_should_replace(connector_type),)@@ -1101,6 +1113,7 @@ async def sync_all_connectors(all_task_ids = []synced_connectors = []skipped_connectors = []+ deleted_only_connectors = []errors = []for connector_type in cloud_connector_types:@@ -1172,7 +1185,7 @@ async def sync_all_connectors(# Reconcile orphans (files deleted at the source) before re-syncing.# sync_all_connectors has no caps or filters, so gating reduces# to the strict checks inside the helper.- await reconcile_orphans_for_connector_type(+ orphan_ids = await reconcile_orphans_for_connector_type(connector_type=connector_type,user_id=user.user_id,connector_service=connector_service,@@ -1181,6 +1194,14 @@ async def sync_all_connectors(existing_file_ids=existing_file_ids,id_field=id_field,)+ if orphan_ids:+ orphan_id_set = set(orphan_ids)+ existing_file_ids = [+ fid for fid in existing_file_ids if fid not in orphan_id_set+ ]+ if not existing_file_ids:+ deleted_only_connectors.append(connector_type)+ continuetask_id = await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,@@ -1224,6 +1245,16 @@ async def sync_all_connectors(errors.append({"connector_type": connector_type, "error": str(e)})if not all_task_ids and not errors:+ if deleted_only_connectors:+ return JSONResponse(+ {+ "status": "no_files",+ "message": "Deleted stale cloud files; no remaining files to sync.",+ "skipped_connectors": skipped_connectors if skipped_connectors else None,+ "deleted_only_connectors": deleted_only_connectors,+ },+ status_code=200,+ )if skipped_connectors:return JSONResponse({@@ -1305,6 +1336,8 @@ async def _preview_orphans_for_connector_type(existing_file_ids=existing_file_ids,id_to_filename=id_to_filename,)+ if orphans is not None:+ synced_count = max(0, synced_count - len(orphans))return orphans, synced_countdiff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex ffdb22264..a438558d3 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -247,7 +247,7 @@ class GoogleDriveConnector(BaseConnector):fileId=target_id,fields=("id, name, mimeType, modifiedTime, createdTime, size, "- "webViewLink, parents, owners, driveId"+ "webViewLink, parents, owners, driveId, trashed"),**self._drives_get_flags,)@@ -342,9 +342,16 @@ class GoogleDriveConnector(BaseConnector):).execute())- return self._resolve_shortcut(meta)- except HttpError:- return None+ if meta.get("trashed"):+ return None+ resolved = self._resolve_shortcut(meta)+ if resolved.get("trashed"):+ return None+ return resolved+ except HttpError as e:+ if getattr(e.resp, "status", None) == 404:+ return None+ raisedef _filter_by_mime(self, items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:"""@@ -415,6 +422,8 @@ class GoogleDriveConnector(BaseConnector):folder_children = self._bfs_expand_folders(folders_to_expand)for meta in folder_children:meta = self._resolve_shortcut(meta)+ if meta.get("trashed"):+ continueif meta.get("id") in seen:continueseen.add(meta["id"])diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.pyindex e7ff1dca8..9e339ca37 100644--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py@@ -9,6 +9,7 @@ The orphan-deletion safety net must:- query either `document_id` or `connector_file_id` depending on the ingest path."""+import jsonimport sysfrom pathlib import Pathfrom types import SimpleNamespace@@ -65,6 +66,10 @@ def _make_session_manager(opensearch_client):return sm+def _json(response):+ return json.loads(response.body.decode())++def _patch_write_client(monkeypatch, *, delete_side_effect=None):write_client = AsyncMock()write_client.delete = AsyncMock(@@ -386,3 +391,162 @@ async def test_document_id_field_used_by_default(monkeypatch):search_body = opensearch_client.search.await_args.kwargs["body"]assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}write_client.delete.assert_awaited_once()+++async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):+ from api.connectors import _preview_orphans_for_connector_type++ monkeypatch.setattr(+ "api.connectors.get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["a", "b", "c", "d", "e", "f"], [], "document_id")),+ )+ monkeypatch.setattr(+ "api.connectors.get_synced_id_to_filename_map",+ AsyncMock(return_value={"b": "b.pdf", "e": "e.pdf"}),+ )+ monkeypatch.setattr(+ "api.connectors.compute_orphans_for_connector_type",+ AsyncMock(+ return_value=[+ {"document_id": "b", "filename": "b.pdf"},+ {"document_id": "e", "filename": "e.pdf"},+ ]+ ),+ )++ orphans, synced_count = await _preview_orphans_for_connector_type(+ connector_type="google_drive",+ user_id="alice",+ connector_service=MagicMock(),+ session_manager=MagicMock(),+ jwt_token="token",+ )++ assert synced_count == 4+ assert orphans == [+ {"document_id": "b", "filename": "b.pdf"},+ {"document_id": "e", "filename": "e.pdf"},+ ]+++async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["a", "b", "c", "d", "e", "f"], [], "document_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=["b", "e"]),+ )++ connection = _make_connection("conn-1")+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ service.get_connector = AsyncMock(return_value=connector)+ service.sync_specific_files = AsyncMock(return_value="task-1")++ response = await connectors_api.connector_sync(+ "google_drive",+ connectors_api.ConnectorSyncBody(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ )++ assert response.status_code == 201+ service.sync_specific_files.assert_awaited_once()+ args = service.sync_specific_files.await_args.args+ assert args[2] == ["a", "c", "d", "f"]+++async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["a", "b"], [], "document_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=["a", "b"]),+ )++ connection = _make_connection("conn-1")+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ service.get_connector = AsyncMock(return_value=connector)+ service.sync_specific_files = AsyncMock()++ response = await connectors_api.connector_sync(+ "google_drive",+ connectors_api.ConnectorSyncBody(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ )++ assert response.status_code == 200+ assert _json(response)["status"] == "no_files"+ service.sync_specific_files.assert_not_awaited()+++async def test_sync_all_returns_deleted_only_without_error(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())++ async def fake_synced_ids(connector_type, *args, **kwargs):+ if connector_type == "google_drive":+ return ["a", "b"], [], "document_id"+ return [], [], "document_id"++ monkeypatch.setattr(connectors_api, "get_synced_file_ids_for_connector", fake_synced_ids)+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=["a", "b"]),+ )++ connection = _make_connection("conn-1")+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ service.get_connector = AsyncMock(return_value=connector)+ service.sync_specific_files = AsyncMock()++ response = await connectors_api.sync_all_connectors(+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ )++ body = _json(response)+ assert response.status_code == 200+ assert body["status"] == "no_files"+ assert body.get("errors") is None+ assert body["deleted_only_connectors"] == ["google_drive"]+ assert "Deleted stale cloud files" in body["message"]+ service.sync_specific_files.assert_not_awaited()diff --git a/tests/unit/connectors/test_google_drive_orphan_detection.py b/tests/unit/connectors/test_google_drive_orphan_detection.pyindex 7c98e6144..3ba83b5a5 100644--- a/tests/unit/connectors/test_google_drive_orphan_detection.py+++ b/tests/unit/connectors/test_google_drive_orphan_detection.py@@ -29,7 +29,7 @@ if str(SRC) not in sys.path:sys.path.insert(0, str(SRC))-def _make_connector(file_ids: list[str]):+def _make_connector(file_ids: list[str] | None = None, folder_ids: list[str] | None = None):"""Build a minimal GoogleDriveConnector via __new__, bypassing __init__.Only the attributes read by `_iter_selected_items` and `authenticate` are@@ -43,7 +43,7 @@ def _make_connector(file_ids: list[str]):client_secret="fake-client-secret",token_file="/tmp/fake-token.json",file_ids=file_ids,- folder_ids=None,+ folder_ids=folder_ids,include_mime_types=None,exclude_mime_types=None,)@@ -107,6 +107,74 @@ def test_missing_file_id_not_in_iter_selected_items():assert result == []+def test_resolve_shortcut_requests_target_trashed_field():+ """Shortcut targets must carry `trashed` so deleted targets are excluded."""+ connector = _make_connector(file_ids=["shortcut-id"])++ connector.service.files.return_value.get.return_value.execute.return_value = {+ "id": "target-id",+ "mimeType": "application/pdf",+ "trashed": False,+ }++ result = connector._resolve_shortcut(+ {+ "id": "shortcut-id",+ "mimeType": "application/vnd.google-apps.shortcut",+ "shortcutDetails": {"targetId": "target-id"},+ }+ )++ assert result["id"] == "target-id"+ _, kwargs = connector.service.files.return_value.get.call_args+ assert "trashed" in kwargs["fields"]+++def test_trashed_shortcut_target_not_in_get_file_meta_by_id():+ """A selected shortcut whose target is in trash must behave as missing."""+ connector = _make_connector(file_ids=["shortcut-id"])+ connector.service.files.return_value.get.return_value.execute.side_effect = [+ {+ "id": "shortcut-id",+ "mimeType": "application/vnd.google-apps.shortcut",+ "shortcutDetails": {"targetId": "target-id"},+ "trashed": False,+ },+ {+ "id": "target-id",+ "mimeType": "application/pdf",+ "trashed": True,+ },+ ]++ assert connector._get_file_meta_by_id("shortcut-id") is None+++def test_trashed_shortcut_target_not_in_expanded_folder_items():+ """Folder expansion must also exclude shortcuts whose targets are trashed."""+ connector = _make_connector(folder_ids=["folder-id"])+ connector.service.files.return_value.get.return_value.execute.return_value = {+ "id": "target-id",+ "mimeType": "application/pdf",+ "trashed": True,+ }++ with patch.object(+ connector,+ "_bfs_expand_folders",+ return_value=[+ {+ "id": "shortcut-id",+ "mimeType": "application/vnd.google-apps.shortcut",+ "shortcutDetails": {"targetId": "target-id"},+ }+ ],+ ):+ result = connector._iter_selected_items()++ assert result == []++# ---------------------------------------------------------------------------# authenticate() — no live API call# ---------------------------------------------------------------------------