Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: handle orphan files and return appropriate responses when syncing connectors (#1715)

Rico Furtadob13f7d46826e
diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.ts
index 214133429..26566131f 100644
--- a/frontend/app/api/mutations/useSyncConnector.ts
+++ b/frontend/app/api/mutations/useSyncConnector.ts
@@ -10,6 +10,7 @@ interface SyncResponse {
connections_synced?: number;
synced_connectors?: string[];
skipped_connectors?: string[];
+ deleted_only_connectors?: string[];
errors?: Array<{ connector_type: string; error: string }> | null;
}
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 4a933b455..665962d87 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -696,8 +696,9 @@ async def connector_sync(
# Reconcile orphans (files deleted at the source) before re-syncing.
# Strict gating: skip when sync is capped — we'd see a partial remote
# listing and delete legitimate files.
+ ids_to_sync = list(existing_file_ids)
if body.max_files is None:
- await reconcile_orphans_for_connector_type(
+ orphan_ids = await reconcile_orphans_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
@@ -706,10 +707,21 @@ async def connector_sync(
existing_file_ids=existing_file_ids,
id_field=id_field,
)
+ if orphan_ids:
+ orphan_id_set = set(orphan_ids)
+ ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]
+ if not ids_to_sync:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": f"Deleted stale {connector_type} files; no remaining files to sync.",
+ },
+ status_code=200,
+ )
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
- existing_file_ids,
+ ids_to_sync,
jwt_token=jwt_token,
replace_duplicates=_connector_sync_should_replace(connector_type),
)
@@ -1101,6 +1113,7 @@ async def sync_all_connectors(
all_task_ids = []
synced_connectors = []
skipped_connectors = []
+ deleted_only_connectors = []
errors = []
for connector_type in cloud_connector_types:
@@ -1172,7 +1185,7 @@ async def sync_all_connectors(
# Reconcile orphans (files deleted at the source) before re-syncing.
# sync_all_connectors has no caps or filters, so gating reduces
# to the strict checks inside the helper.
- await reconcile_orphans_for_connector_type(
+ orphan_ids = await reconcile_orphans_for_connector_type(
connector_type=connector_type,
user_id=user.user_id,
connector_service=connector_service,
@@ -1181,6 +1194,14 @@ async def sync_all_connectors(
existing_file_ids=existing_file_ids,
id_field=id_field,
)
+ if orphan_ids:
+ orphan_id_set = set(orphan_ids)
+ existing_file_ids = [
+ fid for fid in existing_file_ids if fid not in orphan_id_set
+ ]
+ if not existing_file_ids:
+ deleted_only_connectors.append(connector_type)
+ continue
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
@@ -1224,6 +1245,16 @@ async def sync_all_connectors(
errors.append({"connector_type": connector_type, "error": str(e)})
if not all_task_ids and not errors:
+ if deleted_only_connectors:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": "Deleted stale cloud files; no remaining files to sync.",
+ "skipped_connectors": skipped_connectors if skipped_connectors else None,
+ "deleted_only_connectors": deleted_only_connectors,
+ },
+ status_code=200,
+ )
if skipped_connectors:
return JSONResponse(
{
@@ -1305,6 +1336,8 @@ async def _preview_orphans_for_connector_type(
existing_file_ids=existing_file_ids,
id_to_filename=id_to_filename,
)
+ if orphans is not None:
+ synced_count = max(0, synced_count - len(orphans))
return orphans, synced_count
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index ffdb22264..a438558d3 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -247,7 +247,7 @@ class GoogleDriveConnector(BaseConnector):
fileId=target_id,
fields=(
"id, name, mimeType, modifiedTime, createdTime, size, "
- "webViewLink, parents, owners, driveId"
+ "webViewLink, parents, owners, driveId, trashed"
),
**self._drives_get_flags,
)
@@ -342,9 +342,16 @@ class GoogleDriveConnector(BaseConnector):
)
.execute()
)
- return self._resolve_shortcut(meta)
- except HttpError:
- return None
+ if meta.get("trashed"):
+ return None
+ resolved = self._resolve_shortcut(meta)
+ if resolved.get("trashed"):
+ return None
+ return resolved
+ except HttpError as e:
+ if getattr(e.resp, "status", None) == 404:
+ return None
+ raise
def _filter_by_mime(self, items: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
"""
@@ -415,6 +422,8 @@ class GoogleDriveConnector(BaseConnector):
folder_children = self._bfs_expand_folders(folders_to_expand)
for meta in folder_children:
meta = self._resolve_shortcut(meta)
+ if meta.get("trashed"):
+ continue
if meta.get("id") in seen:
continue
seen.add(meta["id"])
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index e7ff1dca8..9e339ca37 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -9,6 +9,7 @@ The orphan-deletion safety net must:
- query either `document_id` or `connector_file_id` depending on the ingest path.
"""
+import json
import sys
from pathlib import Path
from types import SimpleNamespace
@@ -65,6 +66,10 @@ def _make_session_manager(opensearch_client):
return sm
+def _json(response):
+ return json.loads(response.body.decode())
+
+
def _patch_write_client(monkeypatch, *, delete_side_effect=None):
write_client = AsyncMock()
write_client.delete = AsyncMock(
@@ -386,3 +391,162 @@ async def test_document_id_field_used_by_default(monkeypatch):
search_body = opensearch_client.search.await_args.kwargs["body"]
assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}
write_client.delete.assert_awaited_once()
+
+
+async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
+ from api.connectors import _preview_orphans_for_connector_type
+
+ monkeypatch.setattr(
+ "api.connectors.get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["a", "b", "c", "d", "e", "f"], [], "document_id")),
+ )
+ monkeypatch.setattr(
+ "api.connectors.get_synced_id_to_filename_map",
+ AsyncMock(return_value={"b": "b.pdf", "e": "e.pdf"}),
+ )
+ monkeypatch.setattr(
+ "api.connectors.compute_orphans_for_connector_type",
+ AsyncMock(
+ return_value=[
+ {"document_id": "b", "filename": "b.pdf"},
+ {"document_id": "e", "filename": "e.pdf"},
+ ]
+ ),
+ )
+
+ orphans, synced_count = await _preview_orphans_for_connector_type(
+ connector_type="google_drive",
+ user_id="alice",
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ jwt_token="token",
+ )
+
+ assert synced_count == 4
+ assert orphans == [
+ {"document_id": "b", "filename": "b.pdf"},
+ {"document_id": "e", "filename": "e.pdf"},
+ ]
+
+
+async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["a", "b", "c", "d", "e", "f"], [], "document_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=["b", "e"]),
+ )
+
+ connection = _make_connection("conn-1")
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])
+ service.get_connector = AsyncMock(return_value=connector)
+ service.sync_specific_files = AsyncMock(return_value="task-1")
+
+ response = await connectors_api.connector_sync(
+ "google_drive",
+ connectors_api.ConnectorSyncBody(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ args = service.sync_specific_files.await_args.args
+ assert args[2] == ["a", "c", "d", "f"]
+
+
+async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["a", "b"], [], "document_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=["a", "b"]),
+ )
+
+ connection = _make_connection("conn-1")
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])
+ service.get_connector = AsyncMock(return_value=connector)
+ service.sync_specific_files = AsyncMock()
+
+ response = await connectors_api.connector_sync(
+ "google_drive",
+ connectors_api.ConnectorSyncBody(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ )
+
+ assert response.status_code == 200
+ assert _json(response)["status"] == "no_files"
+ service.sync_specific_files.assert_not_awaited()
+
+
+async def test_sync_all_returns_deleted_only_without_error(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+
+ async def fake_synced_ids(connector_type, *args, **kwargs):
+ if connector_type == "google_drive":
+ return ["a", "b"], [], "document_id"
+ return [], [], "document_id"
+
+ monkeypatch.setattr(connectors_api, "get_synced_file_ids_for_connector", fake_synced_ids)
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=["a", "b"]),
+ )
+
+ connection = _make_connection("conn-1")
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])
+ service.get_connector = AsyncMock(return_value=connector)
+ service.sync_specific_files = AsyncMock()
+
+ response = await connectors_api.sync_all_connectors(
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ )
+
+ body = _json(response)
+ assert response.status_code == 200
+ assert body["status"] == "no_files"
+ assert body.get("errors") is None
+ assert body["deleted_only_connectors"] == ["google_drive"]
+ assert "Deleted stale cloud files" in body["message"]
+ service.sync_specific_files.assert_not_awaited()
diff --git a/tests/unit/connectors/test_google_drive_orphan_detection.py b/tests/unit/connectors/test_google_drive_orphan_detection.py
index 7c98e6144..3ba83b5a5 100644
--- a/tests/unit/connectors/test_google_drive_orphan_detection.py
+++ b/tests/unit/connectors/test_google_drive_orphan_detection.py
@@ -29,7 +29,7 @@ if str(SRC) not in sys.path:
sys.path.insert(0, str(SRC))
-def _make_connector(file_ids: list[str]):
+def _make_connector(file_ids: list[str] | None = None, folder_ids: list[str] | None = None):
"""Build a minimal GoogleDriveConnector via __new__, bypassing __init__.
Only the attributes read by `_iter_selected_items` and `authenticate` are
@@ -43,7 +43,7 @@ def _make_connector(file_ids: list[str]):
client_secret="fake-client-secret",
token_file="/tmp/fake-token.json",
file_ids=file_ids,
- folder_ids=None,
+ folder_ids=folder_ids,
include_mime_types=None,
exclude_mime_types=None,
)
@@ -107,6 +107,74 @@ def test_missing_file_id_not_in_iter_selected_items():
assert result == []
+def test_resolve_shortcut_requests_target_trashed_field():
+ """Shortcut targets must carry `trashed` so deleted targets are excluded."""
+ connector = _make_connector(file_ids=["shortcut-id"])
+
+ connector.service.files.return_value.get.return_value.execute.return_value = {
+ "id": "target-id",
+ "mimeType": "application/pdf",
+ "trashed": False,
+ }
+
+ result = connector._resolve_shortcut(
+ {
+ "id": "shortcut-id",
+ "mimeType": "application/vnd.google-apps.shortcut",
+ "shortcutDetails": {"targetId": "target-id"},
+ }
+ )
+
+ assert result["id"] == "target-id"
+ _, kwargs = connector.service.files.return_value.get.call_args
+ assert "trashed" in kwargs["fields"]
+
+
+def test_trashed_shortcut_target_not_in_get_file_meta_by_id():
+ """A selected shortcut whose target is in trash must behave as missing."""
+ connector = _make_connector(file_ids=["shortcut-id"])
+ connector.service.files.return_value.get.return_value.execute.side_effect = [
+ {
+ "id": "shortcut-id",
+ "mimeType": "application/vnd.google-apps.shortcut",
+ "shortcutDetails": {"targetId": "target-id"},
+ "trashed": False,
+ },
+ {
+ "id": "target-id",
+ "mimeType": "application/pdf",
+ "trashed": True,
+ },
+ ]
+
+ assert connector._get_file_meta_by_id("shortcut-id") is None
+
+
+def test_trashed_shortcut_target_not_in_expanded_folder_items():
+ """Folder expansion must also exclude shortcuts whose targets are trashed."""
+ connector = _make_connector(folder_ids=["folder-id"])
+ connector.service.files.return_value.get.return_value.execute.return_value = {
+ "id": "target-id",
+ "mimeType": "application/pdf",
+ "trashed": True,
+ }
+
+ with patch.object(
+ connector,
+ "_bfs_expand_folders",
+ return_value=[
+ {
+ "id": "shortcut-id",
+ "mimeType": "application/vnd.google-apps.shortcut",
+ "shortcutDetails": {"targetId": "target-id"},
+ }
+ ],
+ ):
+ result = connector._iter_selected_items()
+
+ assert result == []
+
+
# ---------------------------------------------------------------------------
# authenticate() — no live API call
# ---------------------------------------------------------------------------