OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: make connector webhook → sync reliable and correctly scoped (SharePoint/OneDrive/Drive) (#1888)
Edwin Jose8aebcd9577a6
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 9de2712e1..ad3320579 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1095,23 +1095,42 @@ async def connector_webhook(# Let the connector handle the webhook and return affected file IDsaffected_files = await connector.handle_webhook(payload)+ user = session_manager.get_user(connection.user_id)+ jwt_token = user.jwt_token if user else None++ # Scope guard: a connection's picker selection is not persisted, so+ # the connector can't filter the change feed to it. Restrict webhook+ # ingestion to files ALREADY indexed for this connector — the same+ # durable scope the no-selection manual sync uses. This stops a stray+ # change (even just opening a file) from auto-ingesting a file the+ # user never selected. Deletions of indexed files still pass (they+ # remain in the index until cleaned up) so chunk-cleanup runs.+ in_scope: list[str] = []if affected_files:+ indexed_ids, _filenames, _id_field = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=connection.user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )+ indexed_set = set(indexed_ids)+ in_scope = [f for f in affected_files if f in indexed_set]++ if in_scope:logger.info("Webhook connection files affected",connection_id=connection.connection_id,affected_count=len(affected_files),+ in_scope_count=len(in_scope),)- user = session_manager.get_user(connection.user_id)- jwt_token = user.jwt_token if user else None-# Trigger incremental sync for affected files. The webhook fires# because the file changed, so replace the indexed copy instead of# tripping the duplicate-filename guard meant for manual uploads.task_id = await connector_service.sync_specific_files(connection.connection_id,connection.user_id,- affected_files,+ in_scope,jwt_token=jwt_token,replace_duplicates=_connector_sync_should_replace(connector_type),)@@ -1119,7 +1138,22 @@ async def connector_webhook(result = {"connection_id": connection.connection_id,"task_id": task_id,- "affected_files": len(affected_files),+ "affected_files": len(in_scope),+ }+ elif affected_files:+ # Changes detected, but none are within the indexed scope —+ # ignore so unselected files are not auto-ingested.+ logger.info(+ "Webhook changes outside synced scope, ignored",+ connection_id=connection.connection_id,+ affected_count=len(affected_files),+ in_scope_count=0,+ )++ result = {+ "connection_id": connection.connection_id,+ "action": "ignored",+ "reason": "out_of_scope",}else:# No specific files identified - just log the webhookdiff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.pyindex 2e171c115..4db428ff0 100644--- a/src/connectors/onedrive/connector.py+++ b/src/connectors/onedrive/connector.py@@ -492,20 +492,35 @@ class OneDriveConnector(BaseConnector):logger.warning(f"No access token available for ACL extraction: {file_id}")return DocumentACL()- # OneDrive permissions API endpoint- permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"-- # Fetch permissions- async with httpx.AsyncClient() as client:- response = await client.get(- permissions_url, headers={"Authorization": f"Bearer {access_token}"}+ # OneDrive permissions API endpoint. A composite "driveId!itemId"+ # must be split into /drives/{driveId}/items/{itemId}; using it+ # verbatim against /me/drive/items/{id} is malformed → empty ACL.+ if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:+ drive_id, item_id = file_id.rsplit("!", 1)+ permissions_url = (+ f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions")+ else:+ permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"- if response.status_code != 200:- logger.warning(f"Failed to fetch permissions for {file_id}: {response.status_code}")- return DocumentACL()+ # Fetch permissions, following pagination for full share lists.+ permissions: list[dict[str, Any]] = []+ async with httpx.AsyncClient() as client:+ url: str | None = permissions_url+ while url:+ response = await client.get(+ url, headers={"Authorization": f"Bearer {access_token}"}+ )+ if response.status_code != 200:+ logger.warning(+ f"Failed to fetch permissions for {file_id}: {response.status_code}"+ )+ return DocumentACL()+ page = response.json()+ permissions.extend(page.get("value", []))+ url = page.get("@odata.nextLink")- permissions_data = response.json()+ permissions_data = {"value": permissions}allowed_users = []allowed_groups = []@@ -1091,6 +1106,20 @@ class OneDriveConnector(BaseConnector):return notifications[0].get("subscriptionId")return None+ @staticmethod+ def _delta_item_file_id(item: dict[str, Any]) -> str:+ """Return the composite ``{driveId}!{itemId}`` id used at ingest time.++ Selected-file listing stores ids as ``driveId!itemId`` (see+ _list_folder_contents), so the webhook delta must emit the same shape+ or the change can't be correlated with the indexed connector_file_id.+ """+ item_id = item.get("id", "")+ if item_id and "!" in item_id:+ return item_id+ drive_id = item.get("parentReference", {}).get("driveId")+ return f"{drive_id}!{item_id}" if drive_id else item_id+async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:"""Handle webhook notification and return affected file IDs.@@ -1129,7 +1158,7 @@ class OneDriveConnector(BaseConnector):# runs its deleted-at-source cleanup# (get_file_content -> 404 -> delete indexed chunks).if "folder" not in item:- affected_files.append(item["id"])+ affected_files.append(self._delta_item_file_id(item))continueif "file" not in item:continue@@ -1145,7 +1174,7 @@ class OneDriveConnector(BaseConnector):continueif modified_at < cutoff:continue- affected_files.append(item["id"])+ affected_files.append(self._delta_item_file_id(item))delta_link = data.get("@odata.deltaLink")if delta_link:diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.pyindex 054284cc0..3d6b15c86 100644--- a/src/connectors/sharepoint/connector.py+++ b/src/connectors/sharepoint/connector.py@@ -553,26 +553,43 @@ class SharePointConnector(BaseConnector):logger.warning(f"No access token available for ACL extraction: {file_id}")return DocumentACL()- # Determine the correct path for permissions API call- # Use the same URL pattern as _get_file_metadata_by_id and list_files- site_info = self._parse_sharepoint_url()- if site_info:- permissions_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/permissions"+ # Determine the correct path for permissions API call. Mirror+ # _get_file_metadata_by_id: a composite "driveId!itemId" id must be+ # split into /drives/{driveId}/items/{itemId}. Using the composite id+ # verbatim against /drive/items/{id} yields a malformed URL → Graph+ # error → empty ACL, which is why shared-user updates never landed.+ if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:+ drive_id, item_id = file_id.rsplit("!", 1)+ permissions_url = (+ f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions"+ )else:- # Fallback to user drive- permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"+ site_info = self._parse_sharepoint_url()+ if site_info:+ permissions_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/permissions"+ else:+ # Fallback to user drive+ permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"- # Fetch permissions+ # Fetch permissions, following pagination so large share lists are+ # captured in full (not just the first page).+ permissions: list[dict[str, Any]] = []async with httpx.AsyncClient() as client:- response = await client.get(- permissions_url, headers={"Authorization": f"Bearer {access_token}"}- )-- if response.status_code != 200:- logger.warning(f"Failed to fetch permissions for {file_id}: {response.status_code}")- return DocumentACL()+ url: str | None = permissions_url+ while url:+ response = await client.get(+ url, headers={"Authorization": f"Bearer {access_token}"}+ )+ if response.status_code != 200:+ logger.warning(+ f"Failed to fetch permissions for {file_id}: {response.status_code}"+ )+ return DocumentACL()+ page = response.json()+ permissions.extend(page.get("value", []))+ url = page.get("@odata.nextLink")- permissions_data = response.json()+ permissions_data = {"value": permissions}allowed_users = []allowed_groups = []@@ -1028,6 +1045,20 @@ class SharePointConnector(BaseConnector):return notifications[0].get("subscriptionId")return None+ @staticmethod+ def _delta_item_file_id(item: dict[str, Any]) -> str:+ """Return the composite ``{driveId}!{itemId}`` id used at ingest time.++ Selected-file listing stores ids as ``driveId!itemId`` (see+ _list_folder_contents), so the webhook delta must emit the same shape+ or the change can't be correlated with the indexed connector_file_id.+ """+ item_id = item.get("id", "")+ if item_id and "!" in item_id:+ return item_id+ drive_id = item.get("parentReference", {}).get("driveId")+ return f"{drive_id}!{item_id}" if drive_id else item_id+async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:"""Handle webhook notification and return affected file IDs.@@ -1074,7 +1105,7 @@ class SharePointConnector(BaseConnector):# runs its deleted-at-source cleanup# (get_file_content -> 404 -> delete indexed chunks).if "folder" not in item:- affected_files.append(item["id"])+ affected_files.append(self._delta_item_file_id(item))continueif "file" not in item:continue@@ -1090,7 +1121,7 @@ class SharePointConnector(BaseConnector):continueif modified_at < cutoff:continue- affected_files.append(item["id"])+ affected_files.append(self._delta_item_file_id(item))delta_link = data.get("@odata.deltaLink")if delta_link:diff --git a/src/dependencies.py b/src/dependencies.pyindex e8807bc5b..15e549781 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -17,6 +17,7 @@ Usage:import asyncioimport dataclasses+import hashlibfrom collections.abc import AsyncIterator, Sequencefrom typing import Optional@@ -29,6 +30,34 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)+# Header names whose values must never be logged verbatim. Logged as a+# redacted fingerprint (length + sha prefix) so a value can be correlated+# across hops without exposing the secret.+_SENSITIVE_HEADERS = {+ "authorization",+ "x-openrag-api-jwt",+ "x-api-key",+ "x-username",+ "cookie",+ "x-ibm-lh-credentials",+}+++def _redact_header(name: str, value: str) -> str:+ """Redact a header value for logging — never emit the raw secret.++ Sensitive headers become ``'<redacted len=NN sha=abcd1234>'`` so values+ can be correlated across hops without exposing the token; non-sensitive+ headers pass through unchanged.+ """+ if not value:+ return ""+ if name.lower() in _SENSITIVE_HEADERS:+ digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:8]+ return f"<redacted len={len(value)} sha={digest}>"+ return value++# Maps composite "{provider}:{subject}" -> SQL users.id. Doubles as the# "we've already ensured a DB row for this user" cache so we don't pay# the round-trip on every authenticated request. Cleared on user/role@@ -865,6 +894,10 @@ async def get_api_key_user_async(# the JWT in get_api_jwt_header() instead.jwt_header = get_jwt_auth_header()raw_jwt = request.headers.get(jwt_header, "")++ safe_headers = {k: _redact_header(k, v) for k, v in request.headers.items()}+ logger.debug("[AUTH] Incoming /v1 request headers (redacted)", headers=safe_headers)+if not (raw_jwt and raw_jwt.strip()):jwt_header = get_api_jwt_header()raw_jwt = request.headers.get(jwt_header, "")@@ -872,6 +905,7 @@ async def get_api_key_user_async("[AUTH] API-key path JWT header lookup",header_name=jwt_header,jwt_present=bool(raw_jwt and raw_jwt.strip()),+ jwt_preview=_redact_header(jwt_header, raw_jwt),)if raw_jwt and raw_jwt.strip():token = raw_jwt[7:].strip() if raw_jwt.startswith("Bearer ") else raw_jwt.strip()@@ -889,6 +923,11 @@ async def get_api_key_user_async(jwt_token=f"Bearer {token}",)_stage_jwt_roles(request, claims, user.user_id)+ logger.debug(+ "[AUTH] API user authenticated via JWT",+ user_id=user.user_id,+ roles=getattr(request.state, "jwt_roles", None),+ )# The forwarded JWT is primary for ALL operations (identity, roles,# and downstream OpenSearch calls, which validate it via OIDC) —# same as the session surface (_get_ibm_user). NOTE (gateway@@ -904,6 +943,7 @@ async def get_api_key_user_async(logger.error("[AUTH] JWT in request header failed verification/decode",header_name=jwt_header,+ jwt_preview=_redact_header(jwt_header, raw_jwt),)raise HTTPException(status_code=401,@@ -928,6 +968,13 @@ async def get_api_key_user_async("[AUTH] JWT not found in request header — run_mode=saas with ""RBAC enabled requires the gateway to forward the user JWT",header_name=jwt_header,+ authorization_present=bool(request.headers.get("authorization")),+ api_jwt_present=bool(request.headers.get(get_api_jwt_header())),+ seen_auth_headers={+ k: _redact_header(k, v)+ for k, v in request.headers.items()+ if k.lower() in _SENSITIVE_HEADERS+ },)raise HTTPException(status_code=401,diff --git a/src/models/processors.py b/src/models/processors.pyindex 5f50dd811..c619c46a8 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -232,6 +232,69 @@ class TaskProcessor:logger.error("Failed to delete existing document", filename=filename, error=str(e))raise+ async def _delete_connector_chunks(+ self,+ file_id: str,+ opensearch_client,+ owner_user_id: str,+ keep_filenames: list[str] | None = None,+ ) -> int:+ """Delete indexed chunks for a connector file by its STABLE id.++ Matches both ``connector_file_id`` (standard path, where ``document_id``+ holds the content hash) and ``document_id`` (Langflow path, where it+ holds the connector id). When ``keep_filenames`` is given, chunks whose+ filename is one of those names are preserved — used to drop only the+ stale OLD-name chunks left behind by a rename, since a connector file+ keeps the same id across renames. Best-effort: logs and returns 0 on+ failure so a cleanup miss never fails the task.+ """+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids++ if not file_id:+ return 0+ try:+ write_client = clients.opensearch+ if write_client is None:+ raise RuntimeError("Backend OpenSearch write client is unavailable")++ query: dict[str, Any] = {+ "bool": {+ "filter": [+ {+ "bool": {+ "should": [+ {"term": {"document_id": file_id}},+ {"term": {"connector_file_id": file_id}},+ ],+ "minimum_should_match": 1,+ }+ },+ {"term": {"owner": owner_user_id}},+ ]+ }+ }+ if keep_filenames:+ query["bool"]["must_not"] = [{"terms": {"filename": keep_filenames}}]++ chunk_ids = await collect_visible_document_ids(+ opensearch_client,+ index=get_index_name(),+ query=query,+ )+ return await delete_document_ids(+ write_client,+ index=get_index_name(),+ document_ids=chunk_ids,+ )+ except Exception as e:+ logger.error(+ "Failed to delete connector chunks",+ file_id=file_id,+ error=str(e),+ )+ return 0+async def process_document_standard(self,file_path: str,@@ -602,7 +665,7 @@ class DocumentFileProcessor(TaskProcessor):if result.get("status") == "error":file_task.status = TaskStatus.FAILED- file_task.error = result.get("error", "Failed to process document")+ file_task.error = result.get("error") or "Failed to process document"file_task.updated_at = time.time()upload_task.failed_files += 1else:@@ -613,7 +676,7 @@ class DocumentFileProcessor(TaskProcessor):except Exception as e:file_task.status = TaskStatus.FAILED- file_task.error = str(e)+ file_task.error = str(e) or repr(e)file_task.updated_at = time.time()upload_task.failed_files += 1raise@@ -718,28 +781,17 @@ class ConnectorFileProcessor(TaskProcessor):except (FileNotFoundError, ValueError) as e:msg = str(e).lower()if "not found" in msg or "404" in msg:- # File gone at source — remove indexed chunks by document_id- # (= connector file_id) so it stops appearing in search/chat.- # Filename rename (e.g. .txt → .md) is irrelevant here.- deleted_chunks = 0- try:- from api.documents import delete_chunks_by_document_ids-- opensearch_client = (- self.document_service.session_manager.get_user_opensearch_client(- self.user_id, self.jwt_token- )- )- deleted_chunks = await delete_chunks_by_document_ids(- [file_id], opensearch_client, get_index_name()- )- except Exception as cleanup_err:- logger.error(- "Failed to clean up chunks for deleted source file",- file_id=file_id,- connection_id=self.connection_id,- error=str(cleanup_err),+ # File gone at source — remove its indexed chunks by the+ # stable connector id (matches both connector_file_id and+ # document_id) so it stops appearing in search/chat.+ opensearch_client = (+ self.document_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token)+ )+ deleted_chunks = await self._delete_connector_chunks(+ file_id, opensearch_client, self.user_id+ )logger.warning("File no longer exists at source — removed from index",@@ -753,6 +805,13 @@ class ConnectorFileProcessor(TaskProcessor):"status": "skipped","reason": "deleted_at_source","deleted_chunks": deleted_chunks,+ # Human-readable message so the tasks view shows this+ # successful cleanup instead of falling back to+ # "Unknown error" for a skip with no message.+ "warning": (+ f"File no longer exists at source; removed from index "+ f"({deleted_chunks} chunk(s) deleted)."+ ),}file_task.updated_at = time.time()upload_task.successful_files += 1@@ -778,6 +837,23 @@ class ConnectorFileProcessor(TaskProcessor):opensearch_client = self.document_service.session_manager.get_user_opensearch_client(self.user_id, self.jwt_token)++ # Rename cleanup: a connector file keeps a stable id across renames,+ # but chunks are keyed by filename/content-hash, so a renamed file+ # leaves its OLD-name chunks orphaned. Drop chunks for this id whose+ # filename differs from the current one. If any were removed (a real+ # rename), force a re-ingest below so the file is re-indexed under+ # the new name instead of short-circuiting as "unchanged".+ renamed = (+ await self._delete_connector_chunks(+ document.id,+ opensearch_client,+ self.user_id,+ keep_filenames=get_filename_aliases(document.filename),+ )+ > 0+ )+if await self.check_filename_exists(document.filename, opensearch_client):if not self.replace_duplicates:file_task.status = TaskStatus.SKIPPED@@ -808,7 +884,7 @@ class ConnectorFileProcessor(TaskProcessor):# Compute hashfile_hash = hash_id(tmp_path)- if await self.check_document_exists(file_hash, opensearch_client):+ if not renamed and await self.check_document_exists(file_hash, opensearch_client):file_task.status = TaskStatus.COMPLETEDfile_task.result = {"status": "unchanged", "id": file_hash}file_task.updated_at = time.time()@@ -962,7 +1038,7 @@ class ConnectorFileProcessor(TaskProcessor):if result.get("status") == "error":file_task.status = TaskStatus.FAILED- file_task.error = result.get("error", "Failed to process document")+ file_task.error = result.get("error") or "Failed to process document"file_task.updated_at = time.time()upload_task.failed_files += 1else:@@ -973,7 +1049,7 @@ class ConnectorFileProcessor(TaskProcessor):except Exception as e:file_task.status = TaskStatus.FAILED- file_task.error = str(e)+ file_task.error = str(e) or repr(e)file_task.updated_at = time.time()upload_task.failed_files += 1raise@@ -1050,7 +1126,7 @@ class S3FileProcessor(TaskProcessor):result["path"] = f"s3://{self.bucket}/{item}"if result.get("status") == "error":file_task.status = TaskStatus.FAILED- file_task.error = result.get("error", "Failed to process document")+ file_task.error = result.get("error") or "Failed to process document"upload_task.failed_files += 1else:file_task.status = TaskStatus.COMPLETED@@ -1059,7 +1135,7 @@ class S3FileProcessor(TaskProcessor):except Exception as e:file_task.status = TaskStatus.FAILED- file_task.error = str(e)+ file_task.error = str(e) or repr(e)upload_task.failed_files += 1finally:file_task.updated_at = time.time()@@ -1211,7 +1287,7 @@ class LangflowFileProcessor(TaskProcessor):except Exception as e:# Update task with failurefile_task.status = TaskStatus.FAILED- file_task.error = str(e)+ file_task.error = str(e) or repr(e)file_task.updated_at = time.time()upload_task.failed_files += 1raisediff --git a/src/services/task_service.py b/src/services/task_service.pyindex 68c2a8440..824cbb4a0 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -577,7 +577,7 @@ class TaskService:if file_task.status == TaskStatus.RUNNING:file_task.status = TaskStatus.FAILEDif not file_task.error:- file_task.error = str(e)+ file_task.error = str(e) or repr(e)logger.error("File processing task exception encountered",diff --git a/tests/unit/connectors/test_webhook_type_alias.py b/tests/unit/connectors/test_webhook_type_alias.pyindex 68d96fe9f..31776e257 100644--- a/tests/unit/connectors/test_webhook_type_alias.py+++ b/tests/unit/connectors/test_webhook_type_alias.py@@ -105,8 +105,19 @@ async def test_webhook_accepts_legacy_google_type(path_type):assert body["connection_id"] == "conn-1"+def _mock_indexed_files(monkeypatch, indexed_ids: list[str]):+ """Stub the already-indexed file set the webhook scope-guard intersects against."""+ import api.connectors as api_connectors++ monkeypatch.setattr(+ api_connectors,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(indexed_ids, [], "connector_file_id")),+ )++@pytest.mark.asyncio-async def test_webhook_sync_replaces_existing_files():+async def test_webhook_sync_replaces_existing_files(monkeypatch):"""A webhook fires because the file changed, so the triggered sync mustreplace the indexed copy instead of failing the duplicate-filename guard."""from api.connectors import connector_webhook@@ -121,6 +132,7 @@ async def test_webhook_sync_replaces_existing_files():handler.handle_webhook = AsyncMock(return_value=["file-1"])service._get_connector = AsyncMock(return_value=handler)service.sync_specific_files = AsyncMock(return_value="task-1")+ _mock_indexed_files(monkeypatch, ["file-1"])request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})response = await connector_webhook(@@ -138,6 +150,111 @@ async def test_webhook_sync_replaces_existing_files():assert sync_kwargs["replace_duplicates"] is True+# ---------------------------------------------------------------------------+# connector_webhook — scope guard (only ingest files already indexed)+# ---------------------------------------------------------------------------+++def _scope_guard_service(connection):+ service = _webhook_service("chan-1", connection)+ service.sync_specific_files = AsyncMock(return_value="task-1")+ return service+++def _scope_guard_connection():+ connection = MagicMock()+ connection.connection_id = "conn-1"+ connection.user_id = "user-1"+ connection.is_active = True+ return connection+++async def test_webhook_ignores_files_outside_indexed_scope(monkeypatch):+ """A change to a file the user never selected (not in the index) must NOT+ be auto-ingested."""+ from api.connectors import connector_webhook++ connection = _scope_guard_connection()+ service = _scope_guard_service(connection)+ service._get_connector = AsyncMock(+ return_value=MagicMock(handle_webhook=AsyncMock(return_value=["unselected-file"]))+ )+ _mock_indexed_files(monkeypatch, ["indexed-file"])++ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})+ response = await connector_webhook(+ "google_drive",+ request,+ connector_service=service,+ session_manager=MagicMock(),+ session=MagicMock(),+ )++ body = json.loads(response.body)+ assert body["status"] == "processed"+ assert body["reason"] == "out_of_scope"+ service.sync_specific_files.assert_not_awaited()+++async def test_webhook_syncs_only_indexed_intersection(monkeypatch):+ """Of several changed files, only those already indexed are synced."""+ from api.connectors import connector_webhook++ connection = _scope_guard_connection()+ service = _scope_guard_service(connection)+ service._get_connector = AsyncMock(+ return_value=MagicMock(+ handle_webhook=AsyncMock(return_value=["indexed-file", "unselected-file"])+ )+ )+ _mock_indexed_files(monkeypatch, ["indexed-file", "other-indexed"])++ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})+ response = await connector_webhook(+ "google_drive",+ request,+ connector_service=service,+ session_manager=MagicMock(),+ session=MagicMock(),+ )++ body = json.loads(response.body)+ assert body["status"] == "processed"+ assert body["affected_files"] == 1+ synced_ids = service.sync_specific_files.await_args.args[2]+ assert synced_ids == ["indexed-file"]+++async def test_webhook_deletion_of_indexed_file_still_syncs(monkeypatch):+ """A deleted file is still indexed at webhook time, so it passes the scope+ guard and reaches sync (which then runs chunk-cleanup via the 404 path)."""+ from api.connectors import connector_webhook++ connection = _scope_guard_connection()+ service = _scope_guard_service(connection)+ service._get_connector = AsyncMock(+ return_value=MagicMock(handle_webhook=AsyncMock(return_value=["deleted-file"]))+ )+ _mock_indexed_files(monkeypatch, ["deleted-file"])++ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})+ response = await connector_webhook(+ "google_drive",+ request,+ connector_service=service,+ session_manager=MagicMock(),+ session=MagicMock(),+ )++ body = json.loads(response.body)+ assert body["task_id"] == "task-1"+ synced_ids = service.sync_specific_files.await_args.args[2]+ assert synced_ids == ["deleted-file"]++@pytest.mark.asyncioasync def test_webhook_unknown_type_is_ignored_not_500():from api.connectors import connector_webhook@@ -292,6 +409,40 @@ async def test_graph_webhook_uses_stored_delta_link(assert connector._delta_link == "https://graph.microsoft.com/v1.0/delta?token=next"[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)+async def test_graph_webhook_emits_composite_drive_item_id(+ tmp_path, monkeypatch, module_path, cls_name, connector_type+):+ """The webhook must return the SAME composite ``driveId!itemId`` id that+ selected-file listing/ingestion stores as connector_file_id — otherwise the+ change can't be correlated with the indexed file and is dropped as+ out-of-scope (the SharePoint webhook bug)."""+ import httpx++ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url=None)+ connector._delta_link = "https://graph.microsoft.com/v1.0/delta?token=prev"++ delta_page = {+ "value": [+ {+ "id": "01ITEM",+ "file": {},+ "parentReference": {"driveId": "b!DRIVE"},+ "lastModifiedDateTime": "2020-01-01T00:00:00Z",+ },+ # A deleted item with a drive-scoped parent also gets the prefix.+ {"id": "01GONE", "deleted": {}, "parentReference": {"driveId": "b!DRIVE"}},+ ],+ "@odata.deltaLink": "https://graph.microsoft.com/v1.0/delta?token=next",+ }+ monkeypatch.setattr(httpx, "AsyncClient", _FakeDeltaClient([delta_page]))++ affected = await connector.handle_webhook(GRAPH_NOTIFICATION)++ assert affected == ["b!DRIVE!01ITEM", "b!DRIVE!01GONE"]++@pytest.mark.asyncio@pytest.mark.parametrize("module_path,cls_name,connector_type", GRAPH_CONNECTORS)async def test_graph_webhook_empty_payload_skips_delta(diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex b30e39f0b..37f527187 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -14,6 +14,22 @@ import pytestfrom connectors.base import ConnectorDocument, DocumentACLfrom models.processors import ConnectorFileProcessorfrom models.tasks import FileTask, TaskStatus, UploadTask+from utils.file_utils import get_filename_aliases++[email protected](autouse=True)+def backend_write_client(monkeypatch):+ """Provide a backend OpenSearch write client (clients.opensearch).++ Chunk deletion (delete_document_by_filename, _delete_connector_chunks) writes+ through this singleton; it is None in unit tests, so patch it to a mock whose+ per-id delete reports success."""+ import config.settings as cfg++ client = AsyncMock()+ client.delete = AsyncMock(return_value={"result": "deleted"})+ monkeypatch.setattr(cfg.clients, "opensearch", client)+ return clientdef _make_document(filename: str = "My Report.pdf") -> ConnectorDocument:@@ -65,15 +81,20 @@ def _wire_connector_processor(document: ConnectorDocument,filename_exists: bool,hash_exists: bool = False,+ rename_stale_exists: bool = False,):opensearch_client = AsyncMock()async def mock_search(index, body, **kwargs):query_str = str(body)+ # Rename-cleanup query is the only one referencing connector_file_id+ # (dual-id should clause); check it first since it also mentions+ # document_id.+ if "connector_file_id" in query_str:+ return _make_search_response(rename_stale_exists)if "document_id" in query_str:return _make_search_response(hash_exists)- else:- return _make_search_response(filename_exists)+ return _make_search_response(filename_exists)opensearch_client.search = mock_searchopensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 3})@@ -122,11 +143,13 @@ async def test_connector_processor_skips_when_filename_exists_and_replace_false(@pytest.mark.asyncio-async def test_connector_processor_deletes_then_ingests_when_replace_true(monkeypatch):+async def test_connector_processor_deletes_then_ingests_when_replace_true(+ monkeypatch, backend_write_client+):monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)processor = _build_connector_processor(replace_duplicates=True)document = _make_document()- opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)+ _wire_connector_processor(processor, document, filename_exists=True)file_task = _make_file_task()upload_task = _make_upload_task()@@ -139,7 +162,9 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true(monkeyawait processor.process_item(upload_task, "file-id-1", file_task)assert file_task.status == TaskStatus.COMPLETED- opensearch_client.delete_by_query.assert_awaited()+ # The existing same-name chunks are removed via the backend write client's+ # per-id delete before re-ingest.+ backend_write_client.delete.assert_awaited()mock_process.assert_awaited_once()# The processed filename must be the original (with space), not a# sanitized variant.@@ -147,11 +172,14 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true(monkey@pytest.mark.asyncio-async def test_connector_processor_deletes_chunks_when_source_returns_404(monkeypatch):+async def test_connector_processor_deletes_chunks_when_source_returns_404(+ monkeypatch, backend_write_client+):"""When the source connector reports the file is gone (404), the processor- must remove the already-indexed chunks queried by connector_file_id — NOT- document_id. Chunks are indexed with document_id=file_hash (SHA of content)- which differs from file_id, so querying document_id would find nothing.+ must remove the already-indexed chunks by the stable connector id, matching+ BOTH connector_file_id (standard path; document_id holds the content hash)+ and document_id (Langflow path). Querying document_id alone would miss+ standard-path chunks."""monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)processor = _build_connector_processor(replace_duplicates=False)@@ -161,8 +189,6 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(monkeyopensearch_client.search = AsyncMock(return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None})- # delete_document_ids issues individual deletes per _id- opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})processor.document_service.session_manager.get_user_opensearch_client.return_value = (opensearch_client)@@ -186,17 +212,17 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(monkeyassert (file_task.result or {}).get("reason") == "deleted_at_source"assert (file_task.result or {}).get("deleted_chunks") == 1assert upload_task.successful_files == 1- opensearch_client.delete.assert_awaited_once()+ # Concrete chunk deletes go through the backend write client.+ backend_write_client.delete.assert_awaited_once()- # The search must target connector_file_id, not document_id.- # document_id holds a SHA content hash that won't match the connector file ID.+ # The cleanup must match BOTH id fields (a single document_id terms query+ # would miss standard-path chunks whose document_id is the content hash).search_call = opensearch_client.search.await_argsquery = search_call.kwargs["body"]["query"]- assert "terms" in query, f"expected a terms query, got: {query}"- assert "connector_file_id" in query["terms"], (- f"cleanup must query connector_file_id, not document_id. Query: {query}"- )- assert query["terms"]["connector_file_id"] == ["file-id-1"]+ shoulds = query["bool"]["filter"][0]["bool"]["should"]+ fields = {next(iter(c["term"])): next(iter(c["term"].values())) for c in shoulds}+ assert fields["connector_file_id"] == "file-id-1"+ assert fields["document_id"] == "file-id-1"@pytest.mark.asyncio@@ -244,15 +270,17 @@ def _wire_langflow_processor(document: ConnectorDocument,filename_exists: bool,hash_exists: bool = False,+ rename_stale_exists: bool = False,):opensearch_client = AsyncMock()async def mock_search(index, body, **kwargs):query_str = str(body)+ if "connector_file_id" in query_str:+ return _make_search_response(rename_stale_exists)if "document_id" in query_str:return _make_search_response(hash_exists)- else:- return _make_search_response(filename_exists)+ return _make_search_response(filename_exists)opensearch_client.search = mock_searchopensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 2})@@ -301,11 +329,13 @@ async def test_langflow_connector_processor_skips_on_filename_collision(monkeypa@pytest.mark.asyncio-async def test_langflow_connector_processor_overwrites_when_replace_true(monkeypatch):+async def test_langflow_connector_processor_overwrites_when_replace_true(+ monkeypatch, backend_write_client+):monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", False)processor = _build_langflow_processor(replace_duplicates=True)document = _make_document()- opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)+ _wire_langflow_processor(processor, document, filename_exists=True)file_task = _make_file_task()upload_task = _make_upload_task()@@ -313,12 +343,16 @@ async def test_langflow_connector_processor_overwrites_when_replace_true(monkeypawait processor.process_item(upload_task, "file-id-1", file_task)assert file_task.status == TaskStatus.COMPLETED- opensearch_client.delete_by_query.assert_awaited()+ # The existing same-name chunks are removed (per-id delete via the backend+ # write client) before re-uploading to Langflow.+ backend_write_client.delete.assert_awaited()processor.connector_service.langflow_service.upload_and_ingest_file.assert_awaited_once()@pytest.mark.asyncio-async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404(monkeypatch):+async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404(+ monkeypatch, backend_write_client+):"""When the source connector reports the file is gone (404), the Langflowprocessor must remove the already-indexed chunks instead of surfacing'File not found: <id>' as a task error. Regression test for SharePoint@@ -331,7 +365,6 @@ async def test_langflow_connector_processor_deletes_chunks_when_source_returns_4opensearch_client.search = AsyncMock(return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None})- opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})processor.document_service.session_manager.get_user_opensearch_client.return_value = (opensearch_client)@@ -361,7 +394,7 @@ async def test_langflow_connector_processor_deletes_chunks_when_source_returns_4assert upload_task.successful_files == 1assert upload_task.failed_files == 0processor.connector_service.langflow_service.upload_and_ingest_file.assert_not_called()- opensearch_client.delete.assert_awaited_once()+ backend_write_client.delete.assert_awaited_once()@pytest.mark.asyncio@@ -381,3 +414,128 @@ async def test_langflow_connector_processor_hash_unchanged_path_preserved(monkeyassert file_task.status == TaskStatus.COMPLETEDassert (file_task.result or {}).get("status") == "unchanged"processor.connector_service.langflow_service.upload_and_ingest_file.assert_not_called()+++# ---------------------------------------------------------------------------+# Rename cleanup: a connector file keeps a stable id across renames, so the+# OLD-name chunks must be removed when it is re-ingested under a new name.+# ---------------------------------------------------------------------------+++async def test_pure_rename_deletes_old_chunks_and_reingests(monkeypatch, backend_write_client):+ """Unchanged content + new name (standard path): the stale old-name chunks+ are deleted and the file is re-ingested under the new name, instead of+ short-circuiting as 'unchanged' and leaving the old name behind."""+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)+ processor = _build_connector_processor(replace_duplicates=True)+ document = _make_document(filename="Renamed.pdf")+ # hash_exists=True (content unchanged) but a stale-name chunk exists.+ _wire_connector_processor(+ processor, document, filename_exists=False, hash_exists=True, rename_stale_exists=True+ )++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(+ processor,+ "process_document_standard",+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),+ ) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ # Old-name chunks removed, and the unchanged short-circuit was bypassed.+ backend_write_client.delete.assert_awaited()+ mock_process.assert_awaited_once()+ assert file_task.status == TaskStatus.COMPLETED+++async def test_no_rename_unchanged_still_short_circuits(monkeypatch, backend_write_client):+ """No rename + unchanged content: nothing is deleted and the file is+ reported 'unchanged' (no needless re-ingest)."""+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)+ processor = _build_connector_processor(replace_duplicates=True)+ document = _make_document()+ _wire_connector_processor(+ processor, document, filename_exists=False, hash_exists=True, rename_stale_exists=False+ )++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(processor, "process_document_standard", new=AsyncMock()) as mock_process:+ await processor.process_item(upload_task, "file-id-1", file_task)++ assert (file_task.result or {}).get("status") == "unchanged"+ mock_process.assert_not_called()+ backend_write_client.delete.assert_not_awaited()+++async def test_rename_cleanup_matches_both_id_fields(monkeypatch, backend_write_client):+ """The rename-cleanup query must match BOTH document_id and+ connector_file_id, and exclude the current filename (and its aliases)."""+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)+ processor = _build_connector_processor(replace_duplicates=True)+ document = _make_document(filename="Renamed.pdf")++ captured = {}+ opensearch_client = AsyncMock()++ async def mock_search(index, body, **kwargs):+ query_str = str(body)+ if "connector_file_id" in query_str:+ captured["query"] = body["query"]+ return {"hits": {"hits": []}}+ return {"hits": {"hits": []}}++ opensearch_client.search = mock_search+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (+ opensearch_client+ )+ connector = MagicMock()+ connector.get_file_content = AsyncMock(return_value=document)+ processor.connector_service.get_connector = AsyncMock(return_value=connector)+ connection = MagicMock()+ connection.connector_type = "sharepoint"+ processor.connector_service.connection_manager = MagicMock()+ processor.connector_service.connection_manager.get_connection = AsyncMock(+ return_value=connection+ )++ with patch.object(+ processor,+ "process_document_standard",+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),+ ):+ await processor.process_item(+ upload_task=_make_upload_task(), item="file-id-1", file_task=_make_file_task()+ )++ shoulds = captured["query"]["bool"]["filter"][0]["bool"]["should"]+ fields = {next(iter(c["term"])) for c in shoulds}+ assert fields == {"document_id", "connector_file_id"}+ excluded = captured["query"]["bool"]["must_not"][0]["terms"]["filename"]+ assert set(get_filename_aliases("Renamed.pdf")).issubset(set(excluded))+++async def test_rename_cleanup_is_best_effort(monkeypatch, backend_write_client):+ """A failure during rename cleanup must not fail the task."""+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)+ processor = _build_connector_processor(replace_duplicates=True)+ document = _make_document()+ _wire_connector_processor(processor, document, filename_exists=False, hash_exists=True)+ # Make the chunk delete blow up; cleanup swallows it and returns 0.+ backend_write_client.delete = AsyncMock(side_effect=RuntimeError("opensearch down"))++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ with patch.object(processor, "process_document_standard", new=AsyncMock()):+ await processor.process_item(upload_task, "file-id-1", file_task)++ # Task still finishes (unchanged short-circuit, since cleanup found/deleted nothing).+ assert file_task.status == TaskStatus.COMPLETED