Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: make connector webhook → sync reliable and correctly scoped (SharePoint/OneDrive/Drive) (#1888)

Edwin Jose8aebcd9577a6
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 9de2712e1..ad3320579 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1095,23 +1095,42 @@ async def connector_webhook(
# Let the connector handle the webhook and return affected file IDs
affected_files = await connector.handle_webhook(payload)
+ user = session_manager.get_user(connection.user_id)
+ jwt_token = user.jwt_token if user else None
+
+ # Scope guard: a connection's picker selection is not persisted, so
+ # the connector can't filter the change feed to it. Restrict webhook
+ # ingestion to files ALREADY indexed for this connector — the same
+ # durable scope the no-selection manual sync uses. This stops a stray
+ # change (even just opening a file) from auto-ingesting a file the
+ # user never selected. Deletions of indexed files still pass (they
+ # remain in the index until cleaned up) so chunk-cleanup runs.
+ in_scope: list[str] = []
if affected_files:
+ indexed_ids, _filenames, _id_field = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=connection.user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+ indexed_set = set(indexed_ids)
+ in_scope = [f for f in affected_files if f in indexed_set]
+
+ if in_scope:
logger.info(
"Webhook connection files affected",
connection_id=connection.connection_id,
affected_count=len(affected_files),
+ in_scope_count=len(in_scope),
)
- user = session_manager.get_user(connection.user_id)
- jwt_token = user.jwt_token if user else None
-
# Trigger incremental sync for affected files. The webhook fires
# because the file changed, so replace the indexed copy instead of
# tripping the duplicate-filename guard meant for manual uploads.
task_id = await connector_service.sync_specific_files(
connection.connection_id,
connection.user_id,
- affected_files,
+ in_scope,
jwt_token=jwt_token,
replace_duplicates=_connector_sync_should_replace(connector_type),
)
@@ -1119,7 +1138,22 @@ async def connector_webhook(
result = {
"connection_id": connection.connection_id,
"task_id": task_id,
- "affected_files": len(affected_files),
+ "affected_files": len(in_scope),
+ }
+ elif affected_files:
+ # Changes detected, but none are within the indexed scope —
+ # ignore so unselected files are not auto-ingested.
+ logger.info(
+ "Webhook changes outside synced scope, ignored",
+ connection_id=connection.connection_id,
+ affected_count=len(affected_files),
+ in_scope_count=0,
+ )
+
+ result = {
+ "connection_id": connection.connection_id,
+ "action": "ignored",
+ "reason": "out_of_scope",
}
else:
# No specific files identified - just log the webhook
diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py
index 2e171c115..4db428ff0 100644
--- a/src/connectors/onedrive/connector.py
+++ b/src/connectors/onedrive/connector.py
@@ -492,20 +492,35 @@ class OneDriveConnector(BaseConnector):
logger.warning(f"No access token available for ACL extraction: {file_id}")
return DocumentACL()
- # OneDrive permissions API endpoint
- permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"
-
- # Fetch permissions
- async with httpx.AsyncClient() as client:
- response = await client.get(
- permissions_url, headers={"Authorization": f"Bearer {access_token}"}
+ # OneDrive permissions API endpoint. A composite "driveId!itemId"
+ # must be split into /drives/{driveId}/items/{itemId}; using it
+ # verbatim against /me/drive/items/{id} is malformed → empty ACL.
+ if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:
+ drive_id, item_id = file_id.rsplit("!", 1)
+ permissions_url = (
+ f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions"
)
+ else:
+ permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"
- if response.status_code != 200:
- logger.warning(f"Failed to fetch permissions for {file_id}: {response.status_code}")
- return DocumentACL()
+ # Fetch permissions, following pagination for full share lists.
+ permissions: list[dict[str, Any]] = []
+ async with httpx.AsyncClient() as client:
+ url: str | None = permissions_url
+ while url:
+ response = await client.get(
+ url, headers={"Authorization": f"Bearer {access_token}"}
+ )
+ if response.status_code != 200:
+ logger.warning(
+ f"Failed to fetch permissions for {file_id}: {response.status_code}"
+ )
+ return DocumentACL()
+ page = response.json()
+ permissions.extend(page.get("value", []))
+ url = page.get("@odata.nextLink")
- permissions_data = response.json()
+ permissions_data = {"value": permissions}
allowed_users = []
allowed_groups = []
@@ -1091,6 +1106,20 @@ class OneDriveConnector(BaseConnector):
return notifications[0].get("subscriptionId")
return None
+ @staticmethod
+ def _delta_item_file_id(item: dict[str, Any]) -> str:
+ """Return the composite ``{driveId}!{itemId}`` id used at ingest time.
+
+ Selected-file listing stores ids as ``driveId!itemId`` (see
+ _list_folder_contents), so the webhook delta must emit the same shape
+ or the change can't be correlated with the indexed connector_file_id.
+ """
+ item_id = item.get("id", "")
+ if item_id and "!" in item_id:
+ return item_id
+ drive_id = item.get("parentReference", {}).get("driveId")
+ return f"{drive_id}!{item_id}" if drive_id else item_id
+
async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification and return affected file IDs.
@@ -1129,7 +1158,7 @@ class OneDriveConnector(BaseConnector):
# runs its deleted-at-source cleanup
# (get_file_content -> 404 -> delete indexed chunks).
if "folder" not in item:
- affected_files.append(item["id"])
+ affected_files.append(self._delta_item_file_id(item))
continue
if "file" not in item:
continue
@@ -1145,7 +1174,7 @@ class OneDriveConnector(BaseConnector):
continue
if modified_at < cutoff:
continue
- affected_files.append(item["id"])
+ affected_files.append(self._delta_item_file_id(item))
delta_link = data.get("@odata.deltaLink")
if delta_link:
diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py
index 054284cc0..3d6b15c86 100644
--- a/src/connectors/sharepoint/connector.py
+++ b/src/connectors/sharepoint/connector.py
@@ -553,26 +553,43 @@ class SharePointConnector(BaseConnector):
logger.warning(f"No access token available for ACL extraction: {file_id}")
return DocumentACL()
- # Determine the correct path for permissions API call
- # Use the same URL pattern as _get_file_metadata_by_id and list_files
- site_info = self._parse_sharepoint_url()
- if site_info:
- permissions_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/permissions"
+ # Determine the correct path for permissions API call. Mirror
+ # _get_file_metadata_by_id: a composite "driveId!itemId" id must be
+ # split into /drives/{driveId}/items/{itemId}. Using the composite id
+ # verbatim against /drive/items/{id} yields a malformed URL → Graph
+ # error → empty ACL, which is why shared-user updates never landed.
+ if "!" in file_id and len(file_id.rsplit("!", 1)) == 2:
+ drive_id, item_id = file_id.rsplit("!", 1)
+ permissions_url = (
+ f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}/permissions"
+ )
else:
- # Fallback to user drive
- permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"
+ site_info = self._parse_sharepoint_url()
+ if site_info:
+ permissions_url = f"{self._graph_base_url}/sites/{site_info['host_name']}:/sites/{site_info['site_name']}:/drive/items/{file_id}/permissions"
+ else:
+ # Fallback to user drive
+ permissions_url = f"{self._graph_base_url}/me/drive/items/{file_id}/permissions"
- # Fetch permissions
+ # Fetch permissions, following pagination so large share lists are
+ # captured in full (not just the first page).
+ permissions: list[dict[str, Any]] = []
async with httpx.AsyncClient() as client:
- response = await client.get(
- permissions_url, headers={"Authorization": f"Bearer {access_token}"}
- )
-
- if response.status_code != 200:
- logger.warning(f"Failed to fetch permissions for {file_id}: {response.status_code}")
- return DocumentACL()
+ url: str | None = permissions_url
+ while url:
+ response = await client.get(
+ url, headers={"Authorization": f"Bearer {access_token}"}
+ )
+ if response.status_code != 200:
+ logger.warning(
+ f"Failed to fetch permissions for {file_id}: {response.status_code}"
+ )
+ return DocumentACL()
+ page = response.json()
+ permissions.extend(page.get("value", []))
+ url = page.get("@odata.nextLink")
- permissions_data = response.json()
+ permissions_data = {"value": permissions}
allowed_users = []
allowed_groups = []
@@ -1028,6 +1045,20 @@ class SharePointConnector(BaseConnector):
return notifications[0].get("subscriptionId")
return None
+ @staticmethod
+ def _delta_item_file_id(item: dict[str, Any]) -> str:
+ """Return the composite ``{driveId}!{itemId}`` id used at ingest time.
+
+ Selected-file listing stores ids as ``driveId!itemId`` (see
+ _list_folder_contents), so the webhook delta must emit the same shape
+ or the change can't be correlated with the indexed connector_file_id.
+ """
+ item_id = item.get("id", "")
+ if item_id and "!" in item_id:
+ return item_id
+ drive_id = item.get("parentReference", {}).get("driveId")
+ return f"{drive_id}!{item_id}" if drive_id else item_id
+
async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification and return affected file IDs.
@@ -1074,7 +1105,7 @@ class SharePointConnector(BaseConnector):
# runs its deleted-at-source cleanup
# (get_file_content -> 404 -> delete indexed chunks).
if "folder" not in item:
- affected_files.append(item["id"])
+ affected_files.append(self._delta_item_file_id(item))
continue
if "file" not in item:
continue
@@ -1090,7 +1121,7 @@ class SharePointConnector(BaseConnector):
continue
if modified_at < cutoff:
continue
- affected_files.append(item["id"])
+ affected_files.append(self._delta_item_file_id(item))
delta_link = data.get("@odata.deltaLink")
if delta_link:
diff --git a/src/dependencies.py b/src/dependencies.py
index e8807bc5b..15e549781 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -17,6 +17,7 @@ Usage:
import asyncio
import dataclasses
+import hashlib
from collections.abc import AsyncIterator, Sequence
from typing import Optional
@@ -29,6 +30,34 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+# Header names whose values must never be logged verbatim. Logged as a
+# redacted fingerprint (length + sha prefix) so a value can be correlated
+# across hops without exposing the secret.
+_SENSITIVE_HEADERS = {
+ "authorization",
+ "x-openrag-api-jwt",
+ "x-api-key",
+ "x-username",
+ "cookie",
+ "x-ibm-lh-credentials",
+}
+
+
+def _redact_header(name: str, value: str) -> str:
+ """Redact a header value for logging — never emit the raw secret.
+
+ Sensitive headers become ``'<redacted len=NN sha=abcd1234>'`` so values
+ can be correlated across hops without exposing the token; non-sensitive
+ headers pass through unchanged.
+ """
+ if not value:
+ return ""
+ if name.lower() in _SENSITIVE_HEADERS:
+ digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:8]
+ return f"<redacted len={len(value)} sha={digest}>"
+ return value
+
+
# Maps composite "{provider}:{subject}" -> SQL users.id. Doubles as the
# "we've already ensured a DB row for this user" cache so we don't pay
# the round-trip on every authenticated request. Cleared on user/role
@@ -865,6 +894,10 @@ async def get_api_key_user_async(
# the JWT in get_api_jwt_header() instead.
jwt_header = get_jwt_auth_header()
raw_jwt = request.headers.get(jwt_header, "")
+
+ safe_headers = {k: _redact_header(k, v) for k, v in request.headers.items()}
+ logger.debug("[AUTH] Incoming /v1 request headers (redacted)", headers=safe_headers)
+
if not (raw_jwt and raw_jwt.strip()):
jwt_header = get_api_jwt_header()
raw_jwt = request.headers.get(jwt_header, "")
@@ -872,6 +905,7 @@ async def get_api_key_user_async(
"[AUTH] API-key path JWT header lookup",
header_name=jwt_header,
jwt_present=bool(raw_jwt and raw_jwt.strip()),
+ jwt_preview=_redact_header(jwt_header, raw_jwt),
)
if raw_jwt and raw_jwt.strip():
token = raw_jwt[7:].strip() if raw_jwt.startswith("Bearer ") else raw_jwt.strip()
@@ -889,6 +923,11 @@ async def get_api_key_user_async(
jwt_token=f"Bearer {token}",
)
_stage_jwt_roles(request, claims, user.user_id)
+ logger.debug(
+ "[AUTH] API user authenticated via JWT",
+ user_id=user.user_id,
+ roles=getattr(request.state, "jwt_roles", None),
+ )
# The forwarded JWT is primary for ALL operations (identity, roles,
# and downstream OpenSearch calls, which validate it via OIDC) —
# same as the session surface (_get_ibm_user). NOTE (gateway
@@ -904,6 +943,7 @@ async def get_api_key_user_async(
logger.error(
"[AUTH] JWT in request header failed verification/decode",
header_name=jwt_header,
+ jwt_preview=_redact_header(jwt_header, raw_jwt),
)
raise HTTPException(
status_code=401,
@@ -928,6 +968,13 @@ async def get_api_key_user_async(
"[AUTH] JWT not found in request header — run_mode=saas with "
"RBAC enabled requires the gateway to forward the user JWT",
header_name=jwt_header,
+ authorization_present=bool(request.headers.get("authorization")),
+ api_jwt_present=bool(request.headers.get(get_api_jwt_header())),
+ seen_auth_headers={
+ k: _redact_header(k, v)
+ for k, v in request.headers.items()
+ if k.lower() in _SENSITIVE_HEADERS
+ },
)
raise HTTPException(
status_code=401,
diff --git a/src/models/processors.py b/src/models/processors.py
index 5f50dd811..c619c46a8 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -232,6 +232,69 @@ class TaskProcessor:
logger.error("Failed to delete existing document", filename=filename, error=str(e))
raise
+ async def _delete_connector_chunks(
+ self,
+ file_id: str,
+ opensearch_client,
+ owner_user_id: str,
+ keep_filenames: list[str] | None = None,
+ ) -> int:
+ """Delete indexed chunks for a connector file by its STABLE id.
+
+ Matches both ``connector_file_id`` (standard path, where ``document_id``
+ holds the content hash) and ``document_id`` (Langflow path, where it
+ holds the connector id). When ``keep_filenames`` is given, chunks whose
+ filename is one of those names are preserved — used to drop only the
+ stale OLD-name chunks left behind by a rename, since a connector file
+ keeps the same id across renames. Best-effort: logs and returns 0 on
+ failure so a cleanup miss never fails the task.
+ """
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+
+ if not file_id:
+ return 0
+ try:
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
+ query: dict[str, Any] = {
+ "bool": {
+ "filter": [
+ {
+ "bool": {
+ "should": [
+ {"term": {"document_id": file_id}},
+ {"term": {"connector_file_id": file_id}},
+ ],
+ "minimum_should_match": 1,
+ }
+ },
+ {"term": {"owner": owner_user_id}},
+ ]
+ }
+ }
+ if keep_filenames:
+ query["bool"]["must_not"] = [{"terms": {"filename": keep_filenames}}]
+
+ chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query=query,
+ )
+ return await delete_document_ids(
+ write_client,
+ index=get_index_name(),
+ document_ids=chunk_ids,
+ )
+ except Exception as e:
+ logger.error(
+ "Failed to delete connector chunks",
+ file_id=file_id,
+ error=str(e),
+ )
+ return 0
+
async def process_document_standard(
self,
file_path: str,
@@ -602,7 +665,7 @@ class DocumentFileProcessor(TaskProcessor):
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
- file_task.error = result.get("error", "Failed to process document")
+ file_task.error = result.get("error") or "Failed to process document"
file_task.updated_at = time.time()
upload_task.failed_files += 1
else:
@@ -613,7 +676,7 @@ class DocumentFileProcessor(TaskProcessor):
except Exception as e:
file_task.status = TaskStatus.FAILED
- file_task.error = str(e)
+ file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
@@ -718,28 +781,17 @@ class ConnectorFileProcessor(TaskProcessor):
except (FileNotFoundError, ValueError) as e:
msg = str(e).lower()
if "not found" in msg or "404" in msg:
- # File gone at source — remove indexed chunks by document_id
- # (= connector file_id) so it stops appearing in search/chat.
- # Filename rename (e.g. .txt → .md) is irrelevant here.
- deleted_chunks = 0
- try:
- from api.documents import delete_chunks_by_document_ids
-
- opensearch_client = (
- self.document_service.session_manager.get_user_opensearch_client(
- self.user_id, self.jwt_token
- )
- )
- deleted_chunks = await delete_chunks_by_document_ids(
- [file_id], opensearch_client, get_index_name()
- )
- except Exception as cleanup_err:
- logger.error(
- "Failed to clean up chunks for deleted source file",
- file_id=file_id,
- connection_id=self.connection_id,
- error=str(cleanup_err),
+ # File gone at source — remove its indexed chunks by the
+ # stable connector id (matches both connector_file_id and
+ # document_id) so it stops appearing in search/chat.
+ opensearch_client = (
+ self.document_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
)
+ )
+ deleted_chunks = await self._delete_connector_chunks(
+ file_id, opensearch_client, self.user_id
+ )
logger.warning(
"File no longer exists at source — removed from index",
@@ -753,6 +805,13 @@ class ConnectorFileProcessor(TaskProcessor):
"status": "skipped",
"reason": "deleted_at_source",
"deleted_chunks": deleted_chunks,
+ # Human-readable message so the tasks view shows this
+ # successful cleanup instead of falling back to
+ # "Unknown error" for a skip with no message.
+ "warning": (
+ f"File no longer exists at source; removed from index "
+ f"({deleted_chunks} chunk(s) deleted)."
+ ),
}
file_task.updated_at = time.time()
upload_task.successful_files += 1
@@ -778,6 +837,23 @@ class ConnectorFileProcessor(TaskProcessor):
opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
self.user_id, self.jwt_token
)
+
+ # Rename cleanup: a connector file keeps a stable id across renames,
+ # but chunks are keyed by filename/content-hash, so a renamed file
+ # leaves its OLD-name chunks orphaned. Drop chunks for this id whose
+ # filename differs from the current one. If any were removed (a real
+ # rename), force a re-ingest below so the file is re-indexed under
+ # the new name instead of short-circuiting as "unchanged".
+ renamed = (
+ await self._delete_connector_chunks(
+ document.id,
+ opensearch_client,
+ self.user_id,
+ keep_filenames=get_filename_aliases(document.filename),
+ )
+ > 0
+ )
+
if await self.check_filename_exists(document.filename, opensearch_client):
if not self.replace_duplicates:
file_task.status = TaskStatus.SKIPPED
@@ -808,7 +884,7 @@ class ConnectorFileProcessor(TaskProcessor):
# Compute hash
file_hash = hash_id(tmp_path)
- if await self.check_document_exists(file_hash, opensearch_client):
+ if not renamed and await self.check_document_exists(file_hash, opensearch_client):
file_task.status = TaskStatus.COMPLETED
file_task.result = {"status": "unchanged", "id": file_hash}
file_task.updated_at = time.time()
@@ -962,7 +1038,7 @@ class ConnectorFileProcessor(TaskProcessor):
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
- file_task.error = result.get("error", "Failed to process document")
+ file_task.error = result.get("error") or "Failed to process document"
file_task.updated_at = time.time()
upload_task.failed_files += 1
else:
@@ -973,7 +1049,7 @@ class ConnectorFileProcessor(TaskProcessor):
except Exception as e:
file_task.status = TaskStatus.FAILED
- file_task.error = str(e)
+ file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
@@ -1050,7 +1126,7 @@ class S3FileProcessor(TaskProcessor):
result["path"] = f"s3://{self.bucket}/{item}"
if result.get("status") == "error":
file_task.status = TaskStatus.FAILED
- file_task.error = result.get("error", "Failed to process document")
+ file_task.error = result.get("error") or "Failed to process document"
upload_task.failed_files += 1
else:
file_task.status = TaskStatus.COMPLETED
@@ -1059,7 +1135,7 @@ class S3FileProcessor(TaskProcessor):
except Exception as e:
file_task.status = TaskStatus.FAILED
- file_task.error = str(e)
+ file_task.error = str(e) or repr(e)
upload_task.failed_files += 1
finally:
file_task.updated_at = time.time()
@@ -1211,7 +1287,7 @@ class LangflowFileProcessor(TaskProcessor):
except Exception as e:
# Update task with failure
file_task.status = TaskStatus.FAILED
- file_task.error = str(e)
+ file_task.error = str(e) or repr(e)
file_task.updated_at = time.time()
upload_task.failed_files += 1
raise
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 68c2a8440..824cbb4a0 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -577,7 +577,7 @@ class TaskService:
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
if not file_task.error:
- file_task.error = str(e)
+ file_task.error = str(e) or repr(e)
logger.error(
"File processing task exception encountered",
diff --git a/tests/unit/connectors/test_webhook_type_alias.py b/tests/unit/connectors/test_webhook_type_alias.py
index 68d96fe9f..31776e257 100644
--- a/tests/unit/connectors/test_webhook_type_alias.py
+++ b/tests/unit/connectors/test_webhook_type_alias.py
@@ -105,8 +105,19 @@ async def test_webhook_accepts_legacy_google_type(path_type):
assert body["connection_id"] == "conn-1"
+def _mock_indexed_files(monkeypatch, indexed_ids: list[str]):
+ """Stub the already-indexed file set the webhook scope-guard intersects against."""
+ import api.connectors as api_connectors
+
+ monkeypatch.setattr(
+ api_connectors,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(indexed_ids, [], "connector_file_id")),
+ )
+
+
@pytest.mark.asyncio
-async def test_webhook_sync_replaces_existing_files():
+async def test_webhook_sync_replaces_existing_files(monkeypatch):
"""A webhook fires because the file changed, so the triggered sync must
replace the indexed copy instead of failing the duplicate-filename guard."""
from api.connectors import connector_webhook
@@ -121,6 +132,7 @@ async def test_webhook_sync_replaces_existing_files():
handler.handle_webhook = AsyncMock(return_value=["file-1"])
service._get_connector = AsyncMock(return_value=handler)
service.sync_specific_files = AsyncMock(return_value="task-1")
+ _mock_indexed_files(monkeypatch, ["file-1"])
request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})
response = await connector_webhook(
@@ -138,6 +150,111 @@ async def test_webhook_sync_replaces_existing_files():
assert sync_kwargs["replace_duplicates"] is True
+# ---------------------------------------------------------------------------
+# connector_webhook — scope guard (only ingest files already indexed)
+# ---------------------------------------------------------------------------
+
+
+def _scope_guard_service(connection):
+ service = _webhook_service("chan-1", connection)
+ service.sync_specific_files = AsyncMock(return_value="task-1")
+ return service
+
+
+def _scope_guard_connection():
+ connection = MagicMock()
+ connection.connection_id = "conn-1"
+ connection.user_id = "user-1"
+ connection.is_active = True
+ return connection
+
+
+async def test_webhook_ignores_files_outside_indexed_scope(monkeypatch):
+ """A change to a file the user never selected (not in the index) must NOT
+ be auto-ingested."""
+ from api.connectors import connector_webhook
+
+ connection = _scope_guard_connection()
+ service = _scope_guard_service(connection)
+ service._get_connector = AsyncMock(
+ return_value=MagicMock(handle_webhook=AsyncMock(return_value=["unselected-file"]))
+ )
+ _mock_indexed_files(monkeypatch, ["indexed-file"])
+
+ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})
+ response = await connector_webhook(
+ "google_drive",
+ request,
+ connector_service=service,
+ session_manager=MagicMock(),
+ session=MagicMock(),
+ )
+
+ body = json.loads(response.body)
+ assert body["status"] == "processed"
+ assert body["reason"] == "out_of_scope"
+ service.sync_specific_files.assert_not_awaited()
+
+
+async def test_webhook_syncs_only_indexed_intersection(monkeypatch):
+ """Of several changed files, only those already indexed are synced."""
+ from api.connectors import connector_webhook
+
+ connection = _scope_guard_connection()
+ service = _scope_guard_service(connection)
+ service._get_connector = AsyncMock(
+ return_value=MagicMock(
+ handle_webhook=AsyncMock(return_value=["indexed-file", "unselected-file"])
+ )
+ )
+ _mock_indexed_files(monkeypatch, ["indexed-file", "other-indexed"])
+
+ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})
+ response = await connector_webhook(
+ "google_drive",
+ request,
+ connector_service=service,
+ session_manager=MagicMock(),
+ session=MagicMock(),
+ )
+
+ body = json.loads(response.body)
+ assert body["status"] == "processed"
+ assert body["affected_files"] == 1
+ synced_ids = service.sync_specific_files.await_args.args[2]
+ assert synced_ids == ["indexed-file"]
+
+
+async def test_webhook_deletion_of_indexed_file_still_syncs(monkeypatch):
+ """A deleted file is still indexed at webhook time, so it passes the scope
+ guard and reaches sync (which then runs chunk-cleanup via the 404 path)."""
+ from api.connectors import connector_webhook
+
+ connection = _scope_guard_connection()
+ service = _scope_guard_service(connection)
+ service._get_connector = AsyncMock(
+ return_value=MagicMock(handle_webhook=AsyncMock(return_value=["deleted-file"]))
+ )
+ _mock_indexed_files(monkeypatch, ["deleted-file"])
+
+ request = _FakeRequest({"content-type": "application/json", "x-goog-channel-id": "chan-1"})
+ response = await connector_webhook(
+ "google_drive",
+ request,
+ connector_service=service,
+ session_manager=MagicMock(),
+ session=MagicMock(),
+ )
+
+ body = json.loads(response.body)
+ assert body["task_id"] == "task-1"
+ synced_ids = service.sync_specific_files.await_args.args[2]
+ assert synced_ids == ["deleted-file"]
+
+
@pytest.mark.asyncio
async def test_webhook_unknown_type_is_ignored_not_500():
from api.connectors import connector_webhook
@@ -292,6 +409,40 @@ async def test_graph_webhook_uses_stored_delta_link(
assert connector._delta_link == "https://graph.microsoft.com/v1.0/delta?token=next"
[email protected]("module_path,cls_name,connector_type", GRAPH_CONNECTORS)
+async def test_graph_webhook_emits_composite_drive_item_id(
+ tmp_path, monkeypatch, module_path, cls_name, connector_type
+):
+ """The webhook must return the SAME composite ``driveId!itemId`` id that
+ selected-file listing/ingestion stores as connector_file_id — otherwise the
+ change can't be correlated with the indexed file and is dropped as
+ out-of-scope (the SharePoint webhook bug)."""
+ import httpx
+
+ connector = _graph_connector(module_path, cls_name, tmp_path, webhook_url=None)
+ connector._delta_link = "https://graph.microsoft.com/v1.0/delta?token=prev"
+
+ delta_page = {
+ "value": [
+ {
+ "id": "01ITEM",
+ "file": {},
+ "parentReference": {"driveId": "b!DRIVE"},
+ "lastModifiedDateTime": "2020-01-01T00:00:00Z",
+ },
+ # A deleted item with a drive-scoped parent also gets the prefix.
+ {"id": "01GONE", "deleted": {}, "parentReference": {"driveId": "b!DRIVE"}},
+ ],
+ "@odata.deltaLink": "https://graph.microsoft.com/v1.0/delta?token=next",
+ }
+ monkeypatch.setattr(httpx, "AsyncClient", _FakeDeltaClient([delta_page]))
+
+ affected = await connector.handle_webhook(GRAPH_NOTIFICATION)
+
+ assert affected == ["b!DRIVE!01ITEM", "b!DRIVE!01GONE"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("module_path,cls_name,connector_type", GRAPH_CONNECTORS)
async def test_graph_webhook_empty_payload_skips_delta(
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
index b30e39f0b..37f527187 100644
--- a/tests/unit/test_connector_processor_filename_dedupe.py
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -14,6 +14,22 @@ import pytest
from connectors.base import ConnectorDocument, DocumentACL
from models.processors import ConnectorFileProcessor
from models.tasks import FileTask, TaskStatus, UploadTask
+from utils.file_utils import get_filename_aliases
+
+
[email protected](autouse=True)
+def backend_write_client(monkeypatch):
+ """Provide a backend OpenSearch write client (clients.opensearch).
+
+ Chunk deletion (delete_document_by_filename, _delete_connector_chunks) writes
+ through this singleton; it is None in unit tests, so patch it to a mock whose
+ per-id delete reports success."""
+ import config.settings as cfg
+
+ client = AsyncMock()
+ client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(cfg.clients, "opensearch", client)
+ return client
def _make_document(filename: str = "My Report.pdf") -> ConnectorDocument:
@@ -65,15 +81,20 @@ def _wire_connector_processor(
document: ConnectorDocument,
filename_exists: bool,
hash_exists: bool = False,
+ rename_stale_exists: bool = False,
):
opensearch_client = AsyncMock()
async def mock_search(index, body, **kwargs):
query_str = str(body)
+ # Rename-cleanup query is the only one referencing connector_file_id
+ # (dual-id should clause); check it first since it also mentions
+ # document_id.
+ if "connector_file_id" in query_str:
+ return _make_search_response(rename_stale_exists)
if "document_id" in query_str:
return _make_search_response(hash_exists)
- else:
- return _make_search_response(filename_exists)
+ return _make_search_response(filename_exists)
opensearch_client.search = mock_search
opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 3})
@@ -122,11 +143,13 @@ async def test_connector_processor_skips_when_filename_exists_and_replace_false(
@pytest.mark.asyncio
-async def test_connector_processor_deletes_then_ingests_when_replace_true(monkeypatch):
+async def test_connector_processor_deletes_then_ingests_when_replace_true(
+ monkeypatch, backend_write_client
+):
monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
processor = _build_connector_processor(replace_duplicates=True)
document = _make_document()
- opensearch_client = _wire_connector_processor(processor, document, filename_exists=True)
+ _wire_connector_processor(processor, document, filename_exists=True)
file_task = _make_file_task()
upload_task = _make_upload_task()
@@ -139,7 +162,9 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true(monkey
await processor.process_item(upload_task, "file-id-1", file_task)
assert file_task.status == TaskStatus.COMPLETED
- opensearch_client.delete_by_query.assert_awaited()
+ # The existing same-name chunks are removed via the backend write client's
+ # per-id delete before re-ingest.
+ backend_write_client.delete.assert_awaited()
mock_process.assert_awaited_once()
# The processed filename must be the original (with space), not a
# sanitized variant.
@@ -147,11 +172,14 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true(monkey
@pytest.mark.asyncio
-async def test_connector_processor_deletes_chunks_when_source_returns_404(monkeypatch):
+async def test_connector_processor_deletes_chunks_when_source_returns_404(
+ monkeypatch, backend_write_client
+):
"""When the source connector reports the file is gone (404), the processor
- must remove the already-indexed chunks queried by connector_file_id — NOT
- document_id. Chunks are indexed with document_id=file_hash (SHA of content)
- which differs from file_id, so querying document_id would find nothing.
+ must remove the already-indexed chunks by the stable connector id, matching
+ BOTH connector_file_id (standard path; document_id holds the content hash)
+ and document_id (Langflow path). Querying document_id alone would miss
+ standard-path chunks.
"""
monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
processor = _build_connector_processor(replace_duplicates=False)
@@ -161,8 +189,6 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(monkey
opensearch_client.search = AsyncMock(
return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}
)
- # delete_document_ids issues individual deletes per _id
- opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
processor.document_service.session_manager.get_user_opensearch_client.return_value = (
opensearch_client
)
@@ -186,17 +212,17 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(monkey
assert (file_task.result or {}).get("reason") == "deleted_at_source"
assert (file_task.result or {}).get("deleted_chunks") == 1
assert upload_task.successful_files == 1
- opensearch_client.delete.assert_awaited_once()
+ # Concrete chunk deletes go through the backend write client.
+ backend_write_client.delete.assert_awaited_once()
- # The search must target connector_file_id, not document_id.
- # document_id holds a SHA content hash that won't match the connector file ID.
+ # The cleanup must match BOTH id fields (a single document_id terms query
+ # would miss standard-path chunks whose document_id is the content hash).
search_call = opensearch_client.search.await_args
query = search_call.kwargs["body"]["query"]
- assert "terms" in query, f"expected a terms query, got: {query}"
- assert "connector_file_id" in query["terms"], (
- f"cleanup must query connector_file_id, not document_id. Query: {query}"
- )
- assert query["terms"]["connector_file_id"] == ["file-id-1"]
+ shoulds = query["bool"]["filter"][0]["bool"]["should"]
+ fields = {next(iter(c["term"])): next(iter(c["term"].values())) for c in shoulds}
+ assert fields["connector_file_id"] == "file-id-1"
+ assert fields["document_id"] == "file-id-1"
@pytest.mark.asyncio
@@ -244,15 +270,17 @@ def _wire_langflow_processor(
document: ConnectorDocument,
filename_exists: bool,
hash_exists: bool = False,
+ rename_stale_exists: bool = False,
):
opensearch_client = AsyncMock()
async def mock_search(index, body, **kwargs):
query_str = str(body)
+ if "connector_file_id" in query_str:
+ return _make_search_response(rename_stale_exists)
if "document_id" in query_str:
return _make_search_response(hash_exists)
- else:
- return _make_search_response(filename_exists)
+ return _make_search_response(filename_exists)
opensearch_client.search = mock_search
opensearch_client.delete_by_query = AsyncMock(return_value={"deleted": 2})
@@ -301,11 +329,13 @@ async def test_langflow_connector_processor_skips_on_filename_collision(monkeypa
@pytest.mark.asyncio
-async def test_langflow_connector_processor_overwrites_when_replace_true(monkeypatch):
+async def test_langflow_connector_processor_overwrites_when_replace_true(
+ monkeypatch, backend_write_client
+):
monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", False)
processor = _build_langflow_processor(replace_duplicates=True)
document = _make_document()
- opensearch_client = _wire_langflow_processor(processor, document, filename_exists=True)
+ _wire_langflow_processor(processor, document, filename_exists=True)
file_task = _make_file_task()
upload_task = _make_upload_task()
@@ -313,12 +343,16 @@ async def test_langflow_connector_processor_overwrites_when_replace_true(monkeyp
await processor.process_item(upload_task, "file-id-1", file_task)
assert file_task.status == TaskStatus.COMPLETED
- opensearch_client.delete_by_query.assert_awaited()
+ # The existing same-name chunks are removed (per-id delete via the backend
+ # write client) before re-uploading to Langflow.
+ backend_write_client.delete.assert_awaited()
processor.connector_service.langflow_service.upload_and_ingest_file.assert_awaited_once()
@pytest.mark.asyncio
-async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404(monkeypatch):
+async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404(
+ monkeypatch, backend_write_client
+):
"""When the source connector reports the file is gone (404), the Langflow
processor must remove the already-indexed chunks instead of surfacing
'File not found: <id>' as a task error. Regression test for SharePoint
@@ -331,7 +365,6 @@ async def test_langflow_connector_processor_deletes_chunks_when_source_returns_4
opensearch_client.search = AsyncMock(
return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}
)
- opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
processor.document_service.session_manager.get_user_opensearch_client.return_value = (
opensearch_client
)
@@ -361,7 +394,7 @@ async def test_langflow_connector_processor_deletes_chunks_when_source_returns_4
assert upload_task.successful_files == 1
assert upload_task.failed_files == 0
processor.connector_service.langflow_service.upload_and_ingest_file.assert_not_called()
- opensearch_client.delete.assert_awaited_once()
+ backend_write_client.delete.assert_awaited_once()
@pytest.mark.asyncio
@@ -381,3 +414,128 @@ async def test_langflow_connector_processor_hash_unchanged_path_preserved(monkey
assert file_task.status == TaskStatus.COMPLETED
assert (file_task.result or {}).get("status") == "unchanged"
processor.connector_service.langflow_service.upload_and_ingest_file.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Rename cleanup: a connector file keeps a stable id across renames, so the
+# OLD-name chunks must be removed when it is re-ingested under a new name.
+# ---------------------------------------------------------------------------
+
+
+async def test_pure_rename_deletes_old_chunks_and_reingests(monkeypatch, backend_write_client):
+ """Unchanged content + new name (standard path): the stale old-name chunks
+ are deleted and the file is re-ingested under the new name, instead of
+ short-circuiting as 'unchanged' and leaving the old name behind."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_connector_processor(replace_duplicates=True)
+ document = _make_document(filename="Renamed.pdf")
+ # hash_exists=True (content unchanged) but a stale-name chunk exists.
+ _wire_connector_processor(
+ processor, document, filename_exists=False, hash_exists=True, rename_stale_exists=True
+ )
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),
+ ) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ # Old-name chunks removed, and the unchanged short-circuit was bypassed.
+ backend_write_client.delete.assert_awaited()
+ mock_process.assert_awaited_once()
+ assert file_task.status == TaskStatus.COMPLETED
+
+
+async def test_no_rename_unchanged_still_short_circuits(monkeypatch, backend_write_client):
+ """No rename + unchanged content: nothing is deleted and the file is
+ reported 'unchanged' (no needless re-ingest)."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_connector_processor(replace_duplicates=True)
+ document = _make_document()
+ _wire_connector_processor(
+ processor, document, filename_exists=False, hash_exists=True, rename_stale_exists=False
+ )
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(processor, "process_document_standard", new=AsyncMock()) as mock_process:
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert (file_task.result or {}).get("status") == "unchanged"
+ mock_process.assert_not_called()
+ backend_write_client.delete.assert_not_awaited()
+
+
+async def test_rename_cleanup_matches_both_id_fields(monkeypatch, backend_write_client):
+ """The rename-cleanup query must match BOTH document_id and
+ connector_file_id, and exclude the current filename (and its aliases)."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_connector_processor(replace_duplicates=True)
+ document = _make_document(filename="Renamed.pdf")
+
+ captured = {}
+ opensearch_client = AsyncMock()
+
+ async def mock_search(index, body, **kwargs):
+ query_str = str(body)
+ if "connector_file_id" in query_str:
+ captured["query"] = body["query"]
+ return {"hits": {"hits": []}}
+ return {"hits": {"hits": []}}
+
+ opensearch_client.search = mock_search
+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(return_value=document)
+ processor.connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "sharepoint"
+ processor.connector_service.connection_manager = MagicMock()
+ processor.connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+
+ with patch.object(
+ processor,
+ "process_document_standard",
+ new=AsyncMock(return_value={"status": "indexed", "id": "hash-1"}),
+ ):
+ await processor.process_item(
+ upload_task=_make_upload_task(), item="file-id-1", file_task=_make_file_task()
+ )
+
+ shoulds = captured["query"]["bool"]["filter"][0]["bool"]["should"]
+ fields = {next(iter(c["term"])) for c in shoulds}
+ assert fields == {"document_id", "connector_file_id"}
+ excluded = captured["query"]["bool"]["must_not"][0]["terms"]["filename"]
+ assert set(get_filename_aliases("Renamed.pdf")).issubset(set(excluded))
+
+
+async def test_rename_cleanup_is_best_effort(monkeypatch, backend_write_client):
+ """A failure during rename cleanup must not fail the task."""
+ monkeypatch.setattr("config.settings.DISABLE_INGEST_WITH_LANGFLOW", True)
+ processor = _build_connector_processor(replace_duplicates=True)
+ document = _make_document()
+ _wire_connector_processor(processor, document, filename_exists=False, hash_exists=True)
+ # Make the chunk delete blow up; cleanup swallows it and returns 0.
+ backend_write_client.delete = AsyncMock(side_effect=RuntimeError("opensearch down"))
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ with patch.object(processor, "process_document_standard", new=AsyncMock()):
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ # Task still finishes (unchanged short-circuit, since cleanup found/deleted nothing).
+ assert file_task.status == TaskStatus.COMPLETED