Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

refactor: Connector sync improvements (#2103)

Mike Fortman24425003e9fe
diff --git a/enhancements/connectors/azure_blob/connector.py b/enhancements/connectors/azure_blob/connector.py
index ad0237efa..f7509553b 100644
--- a/enhancements/connectors/azure_blob/connector.py
+++ b/enhancements/connectors/azure_blob/connector.py
@@ -8,8 +8,9 @@ so the event loop stays responsive while staying consistent with the cached
client lifecycle used by the AWS S3 / IBM COS connectors.
DLS (v1): owner-based. The connector returns an empty-principal ``DocumentACL``;
-``service.process_connector_document`` assigns ``owner`` to the ingesting
-OpenRAG user. Container→group principal mapping is a planned fast-follow.
+``TaskProcessor.process_document_standard`` (``models/processors.py``) assigns
+``owner`` to the ingesting OpenRAG user. Container→group principal mapping is
+a planned fast-follow.
"""
import asyncio
@@ -68,6 +69,7 @@ class AzureBlobConnector(BaseConnector):
CONNECTOR_TYPE = "azure_blob"
CONNECTOR_KIND = "bucket"
+ CHANGE_DETECTION = "timestamp"
CONNECTOR_NAME = "Azure Blob Storage"
CONNECTOR_DESCRIPTION = "Add knowledge from Azure Blob Storage"
CONNECTOR_ICON = "azure-blob"
@@ -334,7 +336,7 @@ class AzureBlobConnector(BaseConnector):
filename = basename(blob_name) or blob_name
# Owner-based DLS (v1): no per-blob principals — ownership is assigned to
- # the ingesting OpenRAG user by service.process_connector_document.
+ # the ingesting OpenRAG user by TaskProcessor.process_document_standard.
acl = DocumentACL(owner=None, allowed_users=[], allowed_groups=[], allowed_principals=[])
return ConnectorDocument(
diff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.py
index dd50eb683..6aef47cc4 100644
--- a/enhancements/connectors/ibm_cos/connector.py
+++ b/enhancements/connectors/ibm_cos/connector.py
@@ -53,6 +53,7 @@ class IBMCOSConnector(BaseConnector):
CONNECTOR_TYPE = "ibm_cos"
CONNECTOR_KIND = "bucket"
+ CHANGE_DETECTION = "timestamp"
CONNECTOR_NAME = "IBM Cloud Object Storage"
CONNECTOR_DESCRIPTION = "Add knowledge from IBM Cloud Object Storage"
CONNECTOR_ICON = "ibm-cos"
diff --git a/src/api/connectors.py b/src/api/connectors.py
index ba83bf843..6f25e7573 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -22,7 +22,6 @@ from services.connector_access_service import (
CONNECTOR_TYPES,
filter_connectors_for_user,
get_access_map,
- is_bucket_connector_type,
is_connector_access_policy_enforced,
is_connector_allowed_for_request,
list_access_for_admin,
@@ -73,8 +72,33 @@ async def _allowed_connector_types_for_request(
def _connector_sync_should_replace(connector_type: str) -> bool:
- """Return True for connector types where sync should replace existing indexed files."""
- return connector_type in ["google_drive", "sharepoint", "onedrive"]
+ """Return True when sync should replace existing indexed files for this
+ connector type, so content changes propagate on re-sync.
+
+ Declared per connector via ``BaseConnector.SYNC_REPLACES_DUPLICATES``
+ (default True) instead of a hardcoded type list, so new connectors get
+ change-propagating sync without touching this module. Unknown types (e.g.
+ stale index docs from a removed enhancement connector) stay conservative
+ and skip duplicates.
+ """
+ from connectors.registry import get_connector_class
+
+ cls = get_connector_class(connector_type)
+ return cls.SYNC_REPLACES_DUPLICATES if cls is not None else False
+
+
+def _connector_uses_timestamp_change_detection(connector_type: str) -> bool:
+ """True when sync should re-ingest only files whose remote modified_time is
+ newer than the stored one (see bucket_changed_file_ids), instead of
+ replacing every indexed file.
+
+ Declared per connector via ``BaseConnector.CHANGE_DETECTION`` — the bucket
+ connectors set ``"timestamp"``; the default is ``"replace_always"``.
+ """
+ from connectors.registry import get_connector_class
+
+ cls = get_connector_class(connector_type)
+ return cls is not None and cls.CHANGE_DETECTION == "timestamp"
def _is_unmapped_keyword_agg_error(err: Exception) -> bool:
@@ -104,8 +128,8 @@ async def get_synced_file_ids_for_connector(
source ID.
- ``filenames``: unique filenames as a fallback when ``file_ids`` is empty.
- ``id_field``: the OpenSearch field name that ``file_ids`` came from
- (``"connector_file_id"`` or ``"document_id"``). Callers must pass this to
- ``delete_orphan_documents`` so deletions target the correct field.
+ (``"connector_file_id"`` or ``"document_id"``). Informational (logging);
+ chunk deletion matches both fields via ``connectors.chunk_cleanup``.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
@@ -473,28 +497,29 @@ async def compute_orphans_for_connector_type(
)
return None
- # Drive the per-id existence check via cfg.file_ids when the
- # connector supports it (SharePoint / OneDrive / Google Drive).
+ # Drive the per-id existence check via list_selected_files when
+ # the connector supports it (SharePoint / OneDrive / Google Drive).
# The flat default of list_files() only returns the *root* listing
# (e.g. /drive/root/children for SharePoint, files-only, no folder
# traversal), so any folder-internal file in OpenSearch would be
# absent from remote_ids and wrongly flagged as an orphan.
- # _list_selected_files iterates each id via _get_file_metadata_by_id
+ # list_selected_files iterates each id via _get_file_metadata_by_id
# and silently drops missing ids, so the resulting `remote_ids` is
# exactly "the subset of existing_file_ids that still exists at
# source" — which is what orphan detection actually needs.
- cfg = getattr(connector, "cfg", None)
- scoped_listing = cfg is not None and bool(existing_file_ids)
+ # cfg is None on bucket connectors (BaseConnector declares it as a
+ # class default), so guard on cfg-is-not-None rather than hasattr:
+ # otherwise bucket connectors route through list_selected_files ->
+ # list_files() (the whole account) instead of the flat listing below.
+ scoped_listing = getattr(connector, "cfg", None) is not None and bool(existing_file_ids)
- original_file_ids = None
- original_folder_ids = None
if scoped_listing:
- original_file_ids = getattr(cfg, "file_ids", None)
- original_folder_ids = getattr(cfg, "folder_ids", None)
- cfg.file_ids = list(existing_file_ids)
- cfg.folder_ids = None
-
- try:
+ page = await connector.list_selected_files(list(existing_file_ids))
+ for f in page.get("files", []):
+ fid = f.get("id")
+ if fid:
+ remote_ids.add(fid)
+ else:
page_token = None
while True:
page = await connector.list_files(page_token=page_token)
@@ -505,10 +530,6 @@ async def compute_orphans_for_connector_type(
page_token = page.get("nextPageToken") or page.get("next_page_token")
if not page_token:
break
- finally:
- if scoped_listing:
- cfg.file_ids = original_file_ids
- cfg.folder_ids = original_folder_ids
except Exception as e:
logger.warning(
"Skipping orphan compute — listing failed",
@@ -531,31 +552,36 @@ async def delete_orphan_documents(
user_id: str,
session_manager,
jwt_token: str | None,
- id_field: str = "document_id",
+ *,
+ connector_type: str | None = None,
+ shared: bool = False,
) -> int:
"""Delete OpenSearch chunks for the given orphan IDs. Returns the number of
chunks deleted (0 on failure).
- ``id_field`` must match the OpenSearch field that ``orphan_ids`` came from —
- either ``"connector_file_id"`` (ConnectorFileProcessor / non-Langflow path)
- or ``"document_id"`` (Langflow path, where document_id holds the connector
- source ID). Pass the value returned as the third element of
- ``get_synced_file_ids_for_connector()``.
+ Deletion matches both id layouts (``connector_file_id`` for the standard
+ ingest path, ``document_id`` for the Langflow path) via
+ ``connectors.chunk_cleanup``, so callers no longer need to track which
+ field the ids came from.
"""
if not orphan_ids:
return 0
- from .documents import delete_chunks_by_document_ids
+ from connectors.chunk_cleanup import delete_connector_file_chunks
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- return await delete_chunks_by_document_ids(
- orphan_ids, opensearch_client, get_index_name(), field=id_field
+ return await delete_connector_file_chunks(
+ orphan_ids,
+ opensearch_client,
+ connector_type=connector_type,
+ owner_user_id=None if shared else user_id,
+ shared=shared,
+ refresh=True,
)
except Exception as e:
logger.error(
"Orphan delete failed",
orphan_count=len(orphan_ids),
- id_field=id_field,
error=str(e),
)
return 0
@@ -568,16 +594,13 @@ async def reconcile_orphans_for_connector_type(
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
- id_field: str = "document_id",
+ *,
+ shared: bool = False,
) -> list[str]:
"""Compute and delete orphans for a connector type. Thin wrapper around
compute_orphans_for_connector_type + delete_orphan_documents preserved for
callers that perform sync immediately after reconcile.
- ``id_field`` must match the OpenSearch field that ``existing_file_ids`` came
- from. Pass the value returned as the third element of
- ``get_synced_file_ids_for_connector()``.
-
Returns the list of orphan file IDs that were deleted (or []).
"""
orphans = await compute_orphans_for_connector_type(
@@ -597,20 +620,138 @@ async def reconcile_orphans_for_connector_type(
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
- id_field=id_field,
+ connector_type=connector_type,
+ shared=shared,
)
logger.info(
"Orphan reconcile complete",
connector_type=connector_type,
orphan_count=len(orphan_ids),
deleted_chunks=deleted,
- id_field=id_field,
)
if deleted <= 0:
return []
return orphan_ids
+async def _sync_existing_connector_files(
+ connector_type: str,
+ working_connection,
+ user_id: str,
+ connector_service,
+ session_manager,
+ jwt_token: str | None,
+ existing_file_ids: list[str],
+ existing_filenames: list[str],
+ id_field: str,
+ *,
+ ingest_settings: dict[str, Any] | None = None,
+ shared: bool = False,
+ reconcile: bool = True,
+ max_files: int | None = None,
+) -> dict[str, Any]:
+ """Re-sync the files already indexed for a connector type — the shared
+ no-selection Sync flow used by both ``connector_sync`` and
+ ``sync_all_connectors``.
+
+ Orphans (deleted at the source) are reconciled first when ``reconcile`` is
+ True, then either timestamp change detection (updates-only re-ingest) or a
+ full re-sync runs depending on the connector's ``CHANGE_DETECTION``
+ capability; connectors with no stored ids fall back to filename filtering.
+
+ Returns an outcome dict the caller maps onto its own response shape:
+ * ``{"outcome": "synced", "task_id": ...}``
+ * ``{"outcome": "deleted_only"}`` — orphan cleanup removed every file;
+ nothing left to sync.
+ * ``{"outcome": "up_to_date"}`` — timestamp change detection found no
+ remote changes.
+ """
+ if existing_file_ids:
+ logger.info(
+ "Syncing specific files by connector file ID",
+ connector_type=connector_type,
+ file_count=len(existing_file_ids),
+ id_field=id_field,
+ )
+ # Reconcile orphans (files deleted at the source) before re-syncing.
+ # Callers gate this: a capped sync sees a partial remote listing and
+ # would delete legitimate files.
+ ids_to_sync = list(existing_file_ids)
+ if reconcile:
+ orphan_ids = await reconcile_orphans_for_connector_type(
+ connector_type=connector_type,
+ user_id=user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ shared=shared,
+ )
+ if orphan_ids:
+ orphan_id_set = set(orphan_ids)
+ ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]
+ if not ids_to_sync:
+ return {"outcome": "deleted_only"}
+ if _connector_uses_timestamp_change_detection(connector_type):
+ # Timestamp Sync is updates-only: re-ingest just the files whose
+ # remote copy is newer than what's indexed (deleting the stale
+ # chunks via replace_duplicates). Unlike replace_duplicates=False
+ # this actually propagates content changes; unlike replacing every
+ # id it skips unchanged files instead of re-fetching the source.
+ connector = await connector_service.get_connector(working_connection.connection_id)
+ changed_ids = await bucket_changed_file_ids(
+ connector,
+ connector_type,
+ user_id,
+ session_manager,
+ jwt_token,
+ ids_to_sync,
+ )
+ if not changed_ids:
+ return {"outcome": "up_to_date"}
+ if max_files is not None:
+ changed_ids = changed_ids[:max_files]
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user_id,
+ changed_ids,
+ jwt_token=jwt_token,
+ ingest_settings=ingest_settings,
+ replace_duplicates=True,
+ shared=shared,
+ )
+ else:
+ if max_files is not None:
+ ids_to_sync = ids_to_sync[:max_files]
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user_id,
+ ids_to_sync,
+ jwt_token=jwt_token,
+ ingest_settings=ingest_settings,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
+ shared=shared,
+ )
+ else:
+ # Fallback: use filename filtering (for Langflow-ingested files without document_id)
+ logger.info(
+ "Syncing files by filename filter (document_id not available)",
+ connector_type=connector_type,
+ filename_count=len(existing_filenames),
+ )
+ task_id = await connector_service.sync_connector_files(
+ working_connection.connection_id,
+ user_id,
+ max_files=max_files,
+ jwt_token=jwt_token,
+ filename_filter=set(existing_filenames),
+ ingest_settings=ingest_settings,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
+ shared=shared,
+ )
+ return {"outcome": "synced", "task_id": task_id}
+
+
class ConnectorSyncBody(BaseModel):
max_files: int | None = None
selected_files: list[Any] | None = None
@@ -675,16 +816,12 @@ async def _expand_selected_connector_files(
file_ids = [f.get("id") for f in selected_files_raw if isinstance(f, dict) and f.get("id")]
expanded_files_info: list[dict[str, Any]] = []
- if file_ids and hasattr(connector, "cfg"):
- # The connector is cached and shared across requests by
- # ConnectionManager, so scope the listing to a per-call copy with its
- # own cfg instead of mutating (and racing on) the shared config.
- scoped_connector = copy.copy(connector)
- scoped_connector.cfg = copy.copy(connector.cfg)
- scoped_connector.cfg.file_ids = file_ids
- scoped_connector.cfg.folder_ids = None
+ # cfg is None on bucket connectors (BaseConnector class default), so guard on
+ # cfg-is-not-None: only cfg-backed connectors expand folders here; bucket
+ # connectors fall through to using selected_files_raw directly below.
+ if file_ids and getattr(connector, "cfg", None) is not None:
try:
- result = await scoped_connector.list_files()
+ result = await connector.list_selected_files(file_ids)
for f in result.get("files", []):
expanded_files_info.append(_connector_file_response(f))
except Exception as e:
@@ -736,21 +873,14 @@ async def _classify_connector_duplicates(
"total_files": len(cleaned_files),
}
- opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- query_body = {
- "size": 10000,
- "query": {"terms": {"filename": list(all_candidates)}},
- "_source": ["filename"],
- }
+ from utils.opensearch_filenames import find_existing_filenames
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
existing_filenames = set()
try:
- response = await opensearch_client.search(index=get_index_name(), body=query_body)
- hits = response.get("hits", {}).get("hits", [])
- for hit in hits:
- fn = hit.get("_source", {}).get("filename")
- if fn:
- existing_filenames.add(fn)
+ existing_filenames = await find_existing_filenames(
+ all_candidates, opensearch_client, get_index_name()
+ )
except Exception as search_err:
if "index_not_found_exception" not in str(search_err):
raise
@@ -1158,6 +1288,14 @@ async def connector_sync(
status_code=400,
)
+ if body.shared and not await has_effective_permission(
+ request, user, rbac, "knowledge:delete:anonymous"
+ ):
+ return JSONResponse(
+ {"error": "Shared sync requires the knowledge:delete:anonymous permission"},
+ status_code=403,
+ )
+
# Get all active connections for this connector type and user
connections = await connector_service.connection_manager.list_connections(
user_id=user.user_id, connector_type=connector_type
@@ -1220,17 +1358,6 @@ async def connector_sync(
# Explicit files selected (e.g., from file picker) - sync those specific files
from .documents import _ensure_index_exists
- if body.shared and body.replace_duplicates:
- if not await has_effective_permission(
- request, user, rbac, "knowledge:delete:anonymous"
- ):
- return JSONResponse(
- {
- "error": "Replacing shared documents requires the knowledge:delete:anonymous permission"
- },
- status_code=403,
- )
-
if not body.replace_duplicates and file_infos:
duplicate_check = await _classify_connector_duplicates(
connector=await connector_service.get_connector(
@@ -1419,101 +1546,40 @@ async def connector_sync(
status_code=200,
)
- # If we have connector file IDs, use sync_specific_files
- # Otherwise, use filename filtering with sync_connector_files
- if existing_file_ids:
- logger.info(
- "Syncing specific files by connector file ID",
- connector_type=connector_type,
- file_count=len(existing_file_ids),
- id_field=id_field,
- )
- # Reconcile orphans (files deleted at the source) before re-syncing.
- # Strict gating: skip when sync is capped — we'd see a partial remote
- # listing and delete legitimate files.
- ids_to_sync = list(existing_file_ids)
- if body.max_files is None:
- orphan_ids = await reconcile_orphans_for_connector_type(
- connector_type=connector_type,
- user_id=user.user_id,
- connector_service=connector_service,
- session_manager=session_manager,
- jwt_token=jwt_token,
- existing_file_ids=existing_file_ids,
- id_field=id_field,
- )
- if orphan_ids:
- orphan_id_set = set(orphan_ids)
- ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]
- if not ids_to_sync:
- return JSONResponse(
- {
- "status": "no_files",
- "message": f"Deleted stale {connector_type} files; no remaining files to sync.",
- },
- status_code=200,
- )
- if is_bucket_connector_type(connector_type):
- # Bucket Sync is updates-only: re-ingest just the blobs whose
- # remote copy is newer than what's indexed (deleting the stale
- # chunks via replace_duplicates). Unlike replace_duplicates=False
- # this actually propagates content changes; unlike replacing every
- # id it skips unchanged blobs instead of re-fetching the container.
- connector = await connector_service.get_connector(
- working_connection.connection_id
- )
- changed_ids = await bucket_changed_file_ids(
- connector,
- connector_type,
- user.user_id,
- session_manager,
- jwt_token,
- ids_to_sync,
- )
- if not changed_ids:
- return JSONResponse(
- {
- "status": "no_files",
- "message": f"All {connector_type} files are already up to date.",
- },
- status_code=200,
- )
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- changed_ids,
- jwt_token=jwt_token,
- ingest_settings=body.settings,
- replace_duplicates=True,
- shared=body.shared,
- )
- else:
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- ids_to_sync,
- jwt_token=jwt_token,
- ingest_settings=body.settings,
- replace_duplicates=_connector_sync_should_replace(connector_type),
- shared=body.shared,
- )
- else:
- # Fallback: use filename filtering (for Langflow-ingested files without document_id)
- logger.info(
- "Syncing files by filename filter (document_id not available)",
- connector_type=connector_type,
- filename_count=len(existing_filenames),
+ sync_result = await _sync_existing_connector_files(
+ connector_type=connector_type,
+ working_connection=working_connection,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ existing_filenames=existing_filenames,
+ id_field=id_field,
+ ingest_settings=body.settings,
+ shared=body.shared,
+ # Strict gating: skip orphan reconcile when sync is capped — we'd
+ # see a partial remote listing and delete legitimate files.
+ reconcile=body.max_files is None,
+ max_files=body.max_files,
+ )
+ if sync_result["outcome"] == "deleted_only":
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": f"Deleted stale {connector_type} files; no remaining files to sync.",
+ },
+ status_code=200,
)
- task_id = await connector_service.sync_connector_files(
- working_connection.connection_id,
- user.user_id,
- max_files=None,
- jwt_token=jwt_token,
- filename_filter=set(existing_filenames),
- ingest_settings=body.settings,
- replace_duplicates=_connector_sync_should_replace(connector_type),
- shared=body.shared,
+ if sync_result["outcome"] == "up_to_date":
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": f"All {connector_type} files are already up to date.",
+ },
+ status_code=200,
)
+ task_id = sync_result["task_id"]
# The bucket_filter path may have already populated task_ids (new + changed
# batches); every other branch sets a single task_id.
if not task_ids:
@@ -2020,82 +2086,25 @@ async def sync_all_connectors(
)
continue
- # Sync using connector file IDs if available, else use filename filter
- if existing_file_ids:
- logger.info(
- "Syncing specific files by connector file ID",
- connector_type=connector_type,
- file_count=len(existing_file_ids),
- id_field=id_field,
- )
- # Reconcile orphans (files deleted at the source) before re-syncing.
- # sync_all_connectors has no caps or filters, so gating reduces
- # to the strict checks inside the helper.
- orphan_ids = await reconcile_orphans_for_connector_type(
- connector_type=connector_type,
- user_id=user.user_id,
- connector_service=connector_service,
- session_manager=session_manager,
- jwt_token=jwt_token,
- existing_file_ids=existing_file_ids,
- id_field=id_field,
- )
- if orphan_ids:
- orphan_id_set = set(orphan_ids)
- existing_file_ids = [
- fid for fid in existing_file_ids if fid not in orphan_id_set
- ]
- if not existing_file_ids:
- deleted_only_connectors.append(connector_type)
- continue
- if is_bucket_connector_type(connector_type):
- # Updates-only change detection (see connector_sync): re-ingest
- # only blobs that changed at source, replacing stale chunks.
- connector = await connector_service.get_connector(
- working_connection.connection_id
- )
- changed_ids = await bucket_changed_file_ids(
- connector,
- connector_type,
- user.user_id,
- session_manager,
- jwt_token,
- existing_file_ids,
- )
- if not changed_ids:
- # Nothing changed at source — already up to date.
- skipped_connectors.append(connector_type)
- continue
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- changed_ids,
- jwt_token=jwt_token,
- replace_duplicates=True,
- )
- else:
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- existing_file_ids,
- jwt_token=jwt_token,
- replace_duplicates=_connector_sync_should_replace(connector_type),
- )
- else:
- # Fallback: use filename filtering
- logger.info(
- "Syncing files by filename filter",
- connector_type=connector_type,
- filename_count=len(existing_filenames),
- )
- task_id = await connector_service.sync_connector_files(
- working_connection.connection_id,
- user.user_id,
- max_files=None,
- jwt_token=jwt_token,
- filename_filter=set(existing_filenames),
- replace_duplicates=_connector_sync_should_replace(connector_type),
- )
+ sync_result = await _sync_existing_connector_files(
+ connector_type=connector_type,
+ working_connection=working_connection,
+ user_id=user.user_id,
+ connector_service=connector_service,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ existing_file_ids=existing_file_ids,
+ existing_filenames=existing_filenames,
+ id_field=id_field,
+ )
+ if sync_result["outcome"] == "deleted_only":
+ deleted_only_connectors.append(connector_type)
+ continue
+ if sync_result["outcome"] == "up_to_date":
+ # Nothing changed at source — already up to date.
+ skipped_connectors.append(connector_type)
+ continue
+ task_id = sync_result["task_id"]
all_task_ids.append(task_id)
synced_connectors.append(connector_type)
diff --git a/src/api/documents.py b/src/api/documents.py
index c614dd7c1..f4901dace 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -198,45 +198,6 @@ async def delete_documents_by_filename_core(
)
-async def delete_chunks_by_document_ids(
- document_ids: list[str],
- opensearch_client,
- index_name: str,
- write_opensearch_client=None,
- field: str = "document_id",
-) -> int:
- """Bulk delete OpenSearch chunks by a keyword field. Returns deleted count.
-
- DLS-safe: enumerate the visible chunk _ids via search, then issue a trusted
- delete per primary id. `delete_by_query` is silently no-opped under DLS
- (returns deleted:N but leaves docs in place).
-
- `field` selects which indexed keyword to match against (default: ``document_id``).
- Pass ``field="connector_file_id"`` to clean up chunks for a deleted connector
- source file when the connector file ID differs from the content hash stored in
- ``document_id``.
- """
- if not document_ids:
- return 0
- from config.settings import clients
- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
-
- chunk_ids = await collect_visible_document_ids(
- opensearch_client,
- index=index_name,
- query={"terms": {field: document_ids}},
- )
- write_client = write_opensearch_client or clients.opensearch
- if write_client is None:
- raise RuntimeError("Backend OpenSearch write client is unavailable")
- return await delete_document_ids(
- write_client,
- index=index_name,
- document_ids=chunk_ids,
- refresh=True,
- )
-
-
async def _ensure_index_exists(jwt_token: str = None):
"""Create the OpenSearch index if it doesn't exist yet."""
from config.settings import clients as app_clients
@@ -263,24 +224,13 @@ async def check_filename_exists(
try:
opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)
- from utils.file_utils import get_filename_aliases
- from utils.opensearch_queries import build_filename_search_body
-
- candidate_filenames = get_filename_aliases(filename)
- if not candidate_filenames:
- return JSONResponse({"exists": False, "filename": filename}, status_code=200)
+ from utils.opensearch_filenames import filename_exists
logger.debug("Checking filename existence", filename=filename, index_name=get_index_name())
exists = False
try:
- for candidate in candidate_filenames:
- search_body = build_filename_search_body(candidate, size=1, source=["filename"])
- response = await opensearch_client.search(index=get_index_name(), body=search_body)
- hits = response.get("hits", {}).get("hits", [])
- if hits:
- exists = True
- break
+ exists = await filename_exists(filename, opensearch_client, get_index_name())
except Exception as search_err:
if "index_not_found_exception" in str(search_err):
logger.info("Index does not exist, creating it now before upload")
diff --git a/src/api/upload.py b/src/api/upload.py
index e2712c8a0..cb2590e4a 100644
--- a/src/api/upload.py
+++ b/src/api/upload.py
@@ -30,6 +30,9 @@ class UploadPathBody(BaseModel):
class UploadBucketBody(BaseModel):
s3_url: str
+ # When True, objects whose filename (S3 key) already exists in the index
+ # replace the indexed copy; when False (default) they are skipped.
+ replace_duplicates: bool = False
async def upload(
@@ -230,6 +233,7 @@ async def upload_bucket(
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
+ replace_duplicates=body.replace_duplicates,
)
task_id = await task_service.create_custom_task(task_user_id, keys, processor)
diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.py
index 12377d2b5..8267a3879 100644
--- a/src/connectors/aws_s3/connector.py
+++ b/src/connectors/aws_s3/connector.py
@@ -47,6 +47,7 @@ class S3Connector(BaseConnector):
CONNECTOR_TYPE = "aws_s3"
CONNECTOR_KIND = "bucket"
+ CHANGE_DETECTION = "timestamp"
CONNECTOR_NAME = "Amazon S3"
CONNECTOR_DESCRIPTION = "Add knowledge from Amazon S3 or any S3-compatible storage"
CONNECTOR_ICON = "aws-s3"
diff --git a/src/connectors/base.py b/src/connectors/base.py
index 48b77b312..9b7a420f9 100644
--- a/src/connectors/base.py
+++ b/src/connectors/base.py
@@ -1,3 +1,4 @@
+import copy
import os
from abc import ABC, abstractmethod
from dataclasses import dataclass
@@ -64,6 +65,23 @@ class BaseConnector(ABC):
# "oauth" connectors authenticate per-user via OAuth env-var credentials.
# "bucket" connectors authenticate via per-connection config dict (HMAC, API key, etc).
CONNECTOR_KIND: str = "oauth"
+ # When a sync re-processes files that are already indexed, pass
+ # replace_duplicates so the indexed copy is replaced and content changes
+ # propagate, instead of being skipped by the duplicate-filename gate.
+ # Default True so a new connector gets change-propagating sync out of the
+ # box (the content-hash "unchanged" short-circuit still avoids re-indexing
+ # identical bytes on the non-Langflow path). Bucket connectors normally
+ # never consult this: their sync paths divert to modified_time change
+ # detection and replace only the blobs that actually changed.
+ SYNC_REPLACES_DUPLICATES: bool = True
+ # How sync decides whether an already-indexed file needs re-ingesting:
+ # "replace_always" — re-process every indexed file on sync, replacing the
+ # indexed copy (the content-hash short-circuit still
+ # skips identical bytes on the non-Langflow path).
+ # "timestamp" — list the source once and re-ingest only files whose
+ # remote modified_time is strictly newer than the
+ # stored one (see bucket_changed_file_ids).
+ CHANGE_DETECTION: str = "replace_always"
# Connector-specific keys in the config dict that must be encrypted at rest.
SECRET_CONFIG_KEYS: tuple = ()
@@ -72,6 +90,11 @@ class BaseConnector(ABC):
CONNECTOR_DESCRIPTION: str = None
CONNECTOR_ICON: str = None # Icon identifier or emoji
+ # Per-connection settings (file_ids/folder_ids selection, etc.). Declared
+ # here so subclasses that set a concrete cfg type still satisfy the base
+ # type; list_selected_files() scopes it via a per-call shallow copy.
+ cfg: Any = None
+
def __init__(self, config: dict[str, Any]):
self.config = config
self._authenticated = False
@@ -175,6 +198,40 @@ class BaseConnector(ABC):
"""List all files. Returns files and next_page_token if any."""
pass
+ async def list_selected_files(
+ self,
+ file_ids: list[str],
+ folder_ids: list[str] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """List specific files/folders by scoping cfg to this call.
+
+ Callers that need list_files() to operate on a specific set of ids
+ previously had to manually save cfg.file_ids / cfg.folder_ids, overwrite
+ them, call list_files(), then restore in a finally block. This method
+ owns that scoping so the caller never has to.
+
+ Connectors are cached and shared across requests by ConnectionManager,
+ so we operate on a per-call shallow copy with its own cfg instead of
+ mutating (and racing on) the shared config.
+ """
+ cfg = getattr(self, "cfg", None)
+ if cfg is None:
+ # No per-call selection state to scope by. Falling back to
+ # list_files() would list the connector's entire account and
+ # silently ignore file_ids, so refuse instead: callers must gate on
+ # `cfg is not None` and use file_ids directly for cfg-less
+ # (bucket) connectors.
+ raise NotImplementedError(
+ f"{type(self).__name__} has no cfg; list_selected_files is only "
+ "supported on cfg-backed connectors (Google Drive/OneDrive/SharePoint)."
+ )
+ scoped = copy.copy(self)
+ scoped.cfg = copy.copy(cfg)
+ scoped.cfg.file_ids = file_ids
+ scoped.cfg.folder_ids = folder_ids
+ return await scoped.list_files(**kwargs)
+
@abstractmethod
async def get_file_content(self, file_id: str) -> ConnectorDocument:
"""Get file content and metadata"""
diff --git a/src/connectors/chunk_cleanup.py b/src/connectors/chunk_cleanup.py
new file mode 100644
index 000000000..6ef2d3aa0
--- /dev/null
+++ b/src/connectors/chunk_cleanup.py
@@ -0,0 +1,110 @@
+"""Single deletion path for a connector file's indexed chunks.
+
+Deletion propagation has multiple *triggers* — orphan reconciliation (file
+missing from a full remote listing), per-file 404 cleanup during sync, rename
+cleanup — but exactly one deletion *semantic*, owned here:
+
+* Chunks are matched by the stable connector file id against BOTH id layouts:
+ the standard ingest path stores it in ``connector_file_id`` (``document_id``
+ holds the content hash), the Langflow path stores it in ``document_id``.
+ Matching a single field misses chunks from the other layout.
+* Optionally owner-scoped: ``owner_user_id`` restricts to that owner's chunks;
+ ``shared=True`` targets ownerless (instance-shared) chunks instead.
+* DLS-safe: visible chunk ``_id``s are enumerated with the caller's
+ (user-scoped) client, then deleted individually with the trusted backend
+ write client — ``delete_by_query`` is silently no-opped under DLS.
+"""
+
+from collections.abc import Iterable
+from typing import Any
+
+
+def build_connector_file_chunks_query(
+ file_ids: Iterable[str],
+ *,
+ connector_type: str | None = None,
+ owner_user_id: str | None = None,
+ shared: bool = False,
+ keep_filenames: Iterable[str] | None = None,
+) -> dict[str, Any]:
+ """Query matching every chunk of the given connector file ids.
+
+ ``connector_type`` narrows the match to a single connector type so IDs
+ that happen to collide across connectors are not affected.
+
+ ``keep_filenames`` excludes chunks indexed under those names — used by
+ rename cleanup to drop only the stale old-name chunks of a file id.
+ """
+ ids = [fid for fid in file_ids if fid]
+ filters: list[dict[str, Any]] = [
+ {
+ "bool": {
+ "should": [
+ {"terms": {"document_id": ids}},
+ {"terms": {"connector_file_id": ids}},
+ # Some indices predate the explicit keyword mapping for
+ # connector_file_id, so it was dynamically mapped as
+ # analyzed text with a .keyword multi-field.
+ {"terms": {"connector_file_id.keyword": ids}},
+ ],
+ "minimum_should_match": 1,
+ }
+ }
+ ]
+ if connector_type:
+ filters.append({"term": {"connector_type": connector_type}})
+ if shared:
+ filters.append({"bool": {"must_not": {"exists": {"field": "owner"}}}})
+ elif owner_user_id:
+ filters.append({"term": {"owner": owner_user_id}})
+ query: dict[str, Any] = {"bool": {"filter": filters}}
+ keep = [name for name in (keep_filenames or []) if name]
+ if keep:
+ query["bool"]["must_not"] = [{"terms": {"filename": keep}}]
+ return query
+
+
+async def delete_connector_file_chunks(
+ file_ids: Iterable[str],
+ opensearch_client,
+ *,
+ connector_type: str | None = None,
+ owner_user_id: str | None = None,
+ shared: bool = False,
+ keep_filenames: Iterable[str] | None = None,
+ refresh: bool = False,
+) -> int:
+ """Delete the indexed chunks of the given connector file ids.
+
+ Returns the number of chunks deleted. Raises on infrastructure failure
+ (missing write client, OpenSearch errors) — callers that must never fail
+ their surrounding task wrap this in their own best-effort handling.
+ """
+ from config.settings import clients, get_index_name
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+
+ ids = [fid for fid in (file_ids or []) if fid]
+ if not ids:
+ return 0
+
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
+ chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query=build_connector_file_chunks_query(
+ ids,
+ connector_type=connector_type,
+ owner_user_id=owner_user_id,
+ shared=shared,
+ keep_filenames=keep_filenames,
+ ),
+ )
+ return await delete_document_ids(
+ write_client,
+ index=get_index_name(),
+ document_ids=chunk_ids,
+ refresh=refresh,
+ )
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 421cb8015..c91f4c95d 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -1,6 +1,6 @@
from typing import Any
-from utils.file_utils import clean_connector_filename, get_file_extension
+from utils.file_utils import clean_connector_filename
from utils.logging_config import get_logger
from .base import BaseConnector, ConnectorDocument
@@ -76,171 +76,6 @@ class ConnectorService:
return self.session_manager.create_jwt_token(user)
return self.session_manager.get_effective_jwt_token(user.user_id, effective_token)
- async def process_connector_document(
- self,
- document: ConnectorDocument,
- owner_user_id: str,
- connector_type: str,
- jwt_token: str = None,
- owner_name: str = None,
- owner_email: str = None,
- ingest_settings: dict[str, Any] | None = None,
- ) -> dict[str, Any]:
- """Process a document from a connector using active processing pipeline"""
- jwt_token = await self._get_effective_sync_jwt(owner_user_id, jwt_token)
-
- from config.settings import DISABLE_INGEST_WITH_LANGFLOW
-
- if not DISABLE_INGEST_WITH_LANGFLOW and self.langflow_service is not None:
- # Process via Langflow pipeline
- from utils.file_utils import langflow_safe_filename_and_mimetype
-
- langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(
- document.filename, document.mimetype
- )
- file_tuple = (langflow_filename, document.content, processed_mimetype)
-
- allowed_users = []
- allowed_groups = []
- allowed_principals = []
- allowed_principal_labels = []
- if document.acl:
- try:
- allowed_users = document.acl.allowed_users or []
- allowed_groups = document.acl.allowed_groups or []
- allowed_principals = document.acl.allowed_principals or []
- allowed_principal_labels = document.acl.allowed_principal_labels or []
- except AttributeError:
- pass
-
- connector_tweak_settings = None
- if isinstance(ingest_settings, dict):
- connector_tweak_settings = dict(ingest_settings)
- connector_tweak_settings.pop("embeddingModel", None)
-
- tweaks = self.langflow_service.merge_ui_ingest_settings_into_tweaks(
- {}, connector_tweak_settings
- )
-
- result = await self.langflow_service.upload_and_ingest_file(
- file_tuple=file_tuple,
- session_id=None,
- tweaks=tweaks,
- settings=ingest_settings,
- jwt_token=jwt_token,
- owner=owner_user_id,
- owner_name=owner_name,
- owner_email=owner_email,
- connector_type=connector_type,
- docling_polling_service=self.task_service.docling_polling_service
- if self.task_service
- else None,
- connector_file_id=document.id,
- source_url=document.source_url,
- allowed_users=allowed_users,
- allowed_groups=allowed_groups,
- allowed_principals=allowed_principals,
- allowed_principal_labels=allowed_principal_labels,
- original_filename=document.filename,
- original_mimetype=document.mimetype,
- )
- return {
- "status": "indexed",
- "filename": document.filename,
- "source_url": document.source_url,
- "document_id": document.id,
- "connector_type": connector_type,
- "langflow_result": result,
- }
- else:
- # Create temporary file from document content
- import os
-
- from utils.file_utils import auto_cleanup_tempfile
-
- suffix = os.path.splitext(document.filename)[1]
- if not suffix:
- suffix = get_file_extension(document.mimetype)
-
- with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
- # Write document content to temp file
- with open(tmp_path, "wb") as f:
- f.write(document.content)
-
- logger.info(
- "[CONNECTOR] Processing document",
- document_id=document.id,
- connector_type=connector_type,
- filename=document.filename,
- )
-
- # Process using consolidated processing pipeline
- from models.processors import TaskProcessor
-
- processor = TaskProcessor(
- document_service=self.document_service,
- models_service=self.models_service,
- docling_service=self.docling_service,
- )
- standard_kwargs: dict[str, Any] = {}
- if isinstance(ingest_settings, dict):
- em = ingest_settings.get("embeddingModel")
- if isinstance(em, str) and em.strip():
- standard_kwargs["embedding_model"] = em.strip()
- for ui_key, param in (
- ("chunkSize", "chunk_size"),
- ("chunkOverlap", "chunk_overlap"),
- ):
- raw = ingest_settings.get(ui_key)
- if raw is not None:
- try:
- standard_kwargs[param] = int(raw)
- except (TypeError, ValueError):
- pass
- if "ocr" in ingest_settings:
- standard_kwargs["ocr"] = bool(ingest_settings["ocr"])
- if "pictureDescriptions" in ingest_settings:
- standard_kwargs["picture_descriptions"] = bool(
- ingest_settings["pictureDescriptions"]
- )
-
- result = await processor.process_document_standard(
- file_path=tmp_path,
- file_hash=document.id,
- owner_user_id=owner_user_id,
- original_filename=document.filename,
- jwt_token=jwt_token,
- owner_name=owner_name,
- owner_email=owner_email,
- file_size=len(document.content) if document.content else 0,
- connector_type=connector_type,
- acl=document.acl,
- connector_file_id=document.id,
- **standard_kwargs,
- )
-
- logger.info(
- "[CONNECTOR] Document processed",
- document_id=document.id,
- status=result.get("status"),
- )
-
- # If successfully indexed or already exists, update the indexed documents with connector metadata
- if result["status"] in ["indexed", "unchanged"]:
- # Update all chunks with connector-specific metadata
- await self._update_connector_metadata(
- document,
- owner_user_id,
- connector_type,
- jwt_token,
- )
-
- return {
- **result,
- "filename": document.filename,
- "source_url": document.source_url,
- }
-
async def _update_connector_metadata(
self,
document: ConnectorDocument,
@@ -546,31 +381,23 @@ class ConnectorService:
connector.set_file_infos(file_infos)
logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")
- # Temporarily set file_ids in the connector's config so list_files() can use them
- # Store the original values to restore later
- original_file_ids = None
- original_folder_ids = None
-
- if hasattr(connector, "cfg"):
- original_file_ids = getattr(connector.cfg, "file_ids", None)
- original_folder_ids = getattr(connector.cfg, "folder_ids", None)
-
expanded_file_ids = file_ids # Default to original IDs
expanded_files_info = []
try:
- # Set the file_ids we want to sync in the connector's config
- if hasattr(connector, "cfg"):
- connector.cfg.file_ids = file_ids
- connector.cfg.folder_ids = None
-
- # Get the expanded list of file IDs (folders will be expanded to their contents)
- # This uses the connector's list_files() which calls _iter_selected_items()
- result = await connector.list_files()
+ # cfg is None on bucket connectors (azure_blob/aws_s3/ibm_cos): they
+ # have no per-call file/folder selection to expand, and file_ids are
+ # already the exact ids to sync. Only cfg-backed connectors
+ # (Google Drive/OneDrive/SharePoint) expand folders here. Guarding on
+ # cfg-is-not-None rather than hasattr is deliberate: BaseConnector
+ # declares cfg=None as a class default, so hasattr is True for every
+ # connector and would route bucket syncs through list_selected_files
+ # -> list_files() (the whole account), discarding the selected ids.
+ if getattr(connector, "cfg", None) is not None:
+ result = await connector.list_selected_files(file_ids)
expanded_files = result.get("files", [])
expanded_file_ids = [f["id"] for f in expanded_files]
- # Save the expanded files info so we can set correct names in the task UI
for f in expanded_files:
expanded_files_info.append(f)
@@ -601,9 +428,6 @@ class ConnectorService:
f"Original IDs: {file_ids}. This may indicate all IDs were folders "
f"with no contents, or files that were filtered out."
)
- # If we have file_infos with download URLs, use original file_ids
- # (OneDrive sharing IDs can't be expanded but can be downloaded directly)
- # Exclude folders — they have no downloadable content on their own.
if file_infos:
non_folder_infos = [f for f in file_infos if not f.get("isFolder")]
non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]
@@ -619,10 +443,8 @@ class ConnectorService:
except Exception as e:
logger.error(f"Failed to expand file_ids via list_files(): {e}")
- # Preserve intentional validation failures (e.g., folders-only selection)
if isinstance(e, ValueError):
raise
- # Fallback path: still exclude known folders when metadata is available
if file_infos:
non_folder_ids = [
f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")
@@ -630,11 +452,6 @@ class ConnectorService:
expanded_file_ids = non_folder_ids or file_ids
else:
expanded_file_ids = file_ids
- finally:
- # Restore original config values
- if hasattr(connector, "cfg"):
- connector.cfg.file_ids = original_file_ids
- connector.cfg.folder_ids = original_folder_ids
# Create custom processor for specific connector files
from models.processors import ConnectorFileProcessor
diff --git a/src/models/processors.py b/src/models/processors.py
index 5f9970912..0cd6498fd 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -21,7 +21,7 @@ from utils.file_utils import (
)
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
-from utils.opensearch_queries import build_filename_search_body, build_replace_filename_query
+from utils.opensearch_queries import build_replace_filename_query
from .tasks import FileTask, TaskStatus, UploadTask
@@ -30,6 +30,8 @@ logger = get_logger(__name__)
DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"
TEXT_PARSER_LABEL = "Text Parser"
+DUPLICATE_FILENAME_WARNING = "A file with this name already exists."
+
if TYPE_CHECKING:
from connectors.base import DocumentACL
@@ -194,41 +196,27 @@ class TaskProcessor:
(default ~1s), and the user-scoped client cannot force an
``indices:admin/refresh`` (it lacks the privilege).
"""
+ from utils.opensearch_filenames import find_existing_filenames
+
max_retries = 3
retry_delay = 1.0
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
return False
- # Keep track of aliases that still need checking across retries.
- # If one alias was already checked successfully with no hits, we avoid
- # re-querying it when another alias fails transiently.
- pending_candidates = list(candidate_filenames)
- # Retry strategy: only retry aliases that have not completed successfully.
- # This avoids re-querying aliases already checked with no hits when a later
- # alias fails transiently (e.g., timeout).
for attempt in range(max_retries):
try:
- i = 0
- while i < len(pending_candidates):
- candidate = pending_candidates[i]
- search_body = build_filename_search_body(candidate, size=1, source=False)
- response = await opensearch_client.search(
- index=get_index_name(), body=search_body
- )
- hits = response.get("hits", {}).get("hits", [])
- if hits:
- return True
- # Successfully checked this alias with no hits; don't
- # re-query it on future retries.
- pending_candidates.pop(i)
- continue
- # All aliases checked with no hits. For post-ingest verification,
- # the document may not be visible yet within the near-real-time
- # refresh window — re-check every alias after a short delay.
+ # One bulk existence check covering every alias — the shared
+ # query semantic used by all duplicate-detection altitudes.
+ if await find_existing_filenames(
+ candidate_filenames, opensearch_client, get_index_name()
+ ):
+ return True
+ # No alias exists. For post-ingest verification, the document
+ # may not be visible yet within the near-real-time refresh
+ # window — re-check after a short delay.
if wait_for_visibility and attempt < max_retries - 1:
- pending_candidates = list(candidate_filenames)
await asyncio.sleep(retry_delay)
retry_delay *= 2
continue
@@ -260,19 +248,84 @@ class TaskProcessor:
retry_delay *= 2 # Exponential backoff
return False
+ async def resolve_duplicate_filename(
+ self,
+ filename: str,
+ opensearch_client,
+ *,
+ replace: bool,
+ owner_user_id: str | None,
+ shared: bool = False,
+ ) -> Literal["proceed", "skip", "replaced"]:
+ """Single duplicate-filename policy shared by every processor.
+
+ Checks whether a document with this filename (or one of its aliases)
+ is already indexed and applies the caller's replace decision:
+
+ * ``"proceed"`` — no duplicate; continue ingestion.
+ * ``"skip"`` — duplicate and ``replace`` is False; the caller
+ should finish via ``mark_duplicate_skipped``.
+ * ``"replaced"`` — duplicate and ``replace`` is True; the existing
+ chunks were deleted and the index refreshed, so
+ ingestion can continue.
+ """
+ if not await self.check_filename_exists(filename, opensearch_client):
+ return "proceed"
+ if not replace:
+ return "skip"
+
+ logger.info(f"Replacing existing document: {filename}")
+ deleted = await self.delete_document_by_filename(
+ filename,
+ opensearch_client,
+ owner_user_id=owner_user_id,
+ shared=shared,
+ )
+ if deleted == 0:
+ logger.warning(
+ "Replacement requested but deletion removed no chunks",
+ filename=filename,
+ )
+ return "skip"
+ # Refresh so the delete is visible before re-ingest. refresh is
+ # index-wide (indices:admin/refresh) and cannot be DLS-scoped, so it
+ # must run under the admin/service client, not the user client.
+ try:
+ await clients.opensearch.indices.refresh(index=get_index_name())
+ except Exception as refresh_error:
+ logger.warning(
+ "Failed to refresh index after delete",
+ error=str(refresh_error),
+ )
+ return "replaced"
+
+ def mark_duplicate_skipped(self, upload_task: UploadTask, file_task: FileTask) -> None:
+ """Uniform terminal state for a duplicate that was not replaced:
+ SKIPPED, counted toward successful files, with a warning the task view
+ surfaces. A declined replacement is a chosen outcome, not an error."""
+ file_task.status = TaskStatus.SKIPPED
+ file_task.error = None
+ file_task.result = {
+ "status": "skipped",
+ "reason": "duplicate_filename",
+ "warning": DUPLICATE_FILENAME_WARNING,
+ }
+ file_task.updated_at = time.time()
+ upload_task.successful_files += 1
+
async def delete_document_by_filename(
self,
filename: str,
opensearch_client,
owner_user_id: str | None = None,
shared: bool = False,
- ) -> None:
- """
- Delete all chunks of a document with the given filename from OpenSearch.
- """
+ ) -> int:
+ """Delete all chunks of a document with the given filename from
+ OpenSearch. Returns the number of chunks deleted."""
from config.settings import clients, get_index_name
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
from utils.opensearch_queries import (
+ build_anonymous_filename_query,
build_owned_filename_query,
build_replace_filename_query,
)
@@ -282,13 +335,20 @@ class TaskProcessor:
if write_client is None:
raise RuntimeError("Backend OpenSearch write client is unavailable")
- deleted_count = 0
if not owner_user_id:
- logger.warning(
- "Skipped delete_by_filename because owner_user_id is missing",
- filename=filename,
- )
- return
+ if shared:
+
+ def build_query(fname, _owner):
+ return build_anonymous_filename_query(fname)
+ else:
+ logger.warning(
+ "Skipped delete_by_filename because owner_user_id is missing",
+ filename=filename,
+ )
+ return 0
+
+ else:
+ build_query = build_replace_filename_query if shared else build_owned_filename_query
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
@@ -296,14 +356,9 @@ class TaskProcessor:
"Skipped delete_by_filename because filename input is empty",
filename=filename,
)
- return
-
- # When shared=True the document being replaced may have previously
- # been ingested without an owner field (also shared), so the normal
- # owner-scoped query would miss those chunks. Use a broader query
- # that covers both owned and ownerless chunks for this filename.
- build_query = build_replace_filename_query if shared else build_owned_filename_query
+ return 0
+ deleted_count = 0
for candidate in candidate_filenames:
document_ids = await collect_visible_document_ids(
opensearch_client,
@@ -318,6 +373,7 @@ class TaskProcessor:
logger.info(
"Deleted existing document chunks", filename=filename, deleted_count=deleted_count
)
+ return deleted_count
except Exception as e:
logger.error("Failed to delete existing document", filename=filename, error=str(e))
@@ -330,64 +386,31 @@ class TaskProcessor:
owner_user_id: str,
keep_filenames: list[str] | None = None,
shared: bool = False,
+ connector_type: str | None = None,
) -> int:
"""Delete indexed chunks for a connector file by its STABLE id.
- Matches both ``connector_file_id`` (standard path, where ``document_id``
- holds the content hash) and ``document_id`` (Langflow path, where it
- holds the connector id). When ``keep_filenames`` is given, chunks whose
- filename is one of those names are preserved — used to drop only the
- stale OLD-name chunks left behind by a rename, since a connector file
- keeps the same id across renames. Best-effort: logs and returns 0 on
- failure so a cleanup miss never fails the task.
+ Deletion semantics (dual-field id match, connector/owner/shared scoping,
+ rename ``keep_filenames``) live in ``connectors.chunk_cleanup``. This
+ wrapper is best-effort: logs and returns 0 on failure so a cleanup miss
+ never fails the task.
+
+ ``connector_type`` scopes the match to one connector type — the same
+ value the chunks were indexed under — so an id that collides with a
+ different connector's id can't take its chunks down with it.
"""
- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+ from connectors.chunk_cleanup import delete_connector_file_chunks
if not file_id:
return 0
try:
- write_client = clients.opensearch
- if write_client is None:
- raise RuntimeError("Backend OpenSearch write client is unavailable")
-
- owner_filter = (
- {"bool": {"must_not": {"exists": {"field": "owner"}}}}
- if shared
- else {"term": {"owner": owner_user_id}}
- )
- query: dict[str, Any] = {
- "bool": {
- "filter": [
- {
- "bool": {
- "should": [
- {"term": {"document_id": file_id}},
- {"term": {"connector_file_id": file_id}},
- # Some deployments' indices predate this field's
- # addition to the explicit mapping, so it was
- # dynamically mapped as analyzed text with a
- # `.keyword` multi-field instead of `keyword`.
- {"term": {"connector_file_id.keyword": file_id}},
- ],
- "minimum_should_match": 1,
- }
- },
- owner_filter,
- ]
- }
- }
- if keep_filenames:
- query["bool"]["must_not"] = [{"terms": {"filename": keep_filenames}}]
-
- chunk_ids = await collect_visible_document_ids(
+ return await delete_connector_file_chunks(
+ [file_id],
opensearch_client,
- index=get_index_name(),
- query=query,
- )
- return await delete_document_ids(
- write_client,
- index=get_index_name(),
- document_ids=chunk_ids,
+ connector_type=connector_type,
+ owner_user_id=owner_user_id,
+ shared=shared,
+ keep_filenames=keep_filenames,
)
except Exception as e:
logger.error(
@@ -728,35 +751,15 @@ class DocumentFileProcessor(TaskProcessor):
self.owner_user_id, self.jwt_token
)
- filename_exists = await self.check_filename_exists(original_filename, opensearch_client)
-
- if filename_exists and not self.replace_duplicates:
- # Duplicate exists and user hasn't confirmed replacement
- file_task.status = TaskStatus.FAILED
- file_task.error = f"File with name '{original_filename}' already exists"
- file_task.updated_at = time.time()
- upload_task.failed_files += 1
+ duplicate_action = await self.resolve_duplicate_filename(
+ original_filename,
+ opensearch_client,
+ replace=self.replace_duplicates,
+ owner_user_id=self.owner_user_id,
+ )
+ if duplicate_action == "skip":
+ self.mark_duplicate_skipped(upload_task, file_task)
return
- elif filename_exists and self.replace_duplicates:
- # Delete existing document before uploading new one
- logger.info(f"Replacing existing document: {original_filename}")
- await self.delete_document_by_filename(
- original_filename,
- opensearch_client,
- owner_user_id=self.owner_user_id,
- )
- # Refresh index to make deletion visible before processing.
- # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,
- # so it must run under the admin/service client, not the user client.
- from config.settings import get_index_name
-
- try:
- await clients.opensearch.indices.refresh(index=get_index_name())
- except Exception as refresh_error:
- logger.warning(
- "Failed to refresh index after delete",
- error=str(refresh_error),
- )
# Compute hash
file_hash = hash_id(item)
@@ -1007,7 +1010,11 @@ class ConnectorFileProcessor(TaskProcessor):
)
)
deleted_chunks = await self._delete_connector_chunks(
- file_id, opensearch_client, self.user_id, shared=self.shared
+ file_id,
+ opensearch_client,
+ self.user_id,
+ shared=self.shared,
+ connector_type=connector_type,
)
logger.warning(
@@ -1055,6 +1062,18 @@ class ConnectorFileProcessor(TaskProcessor):
self.user_id, self.jwt_token
)
+ duplicate_action = await self.resolve_duplicate_filename(
+ file_task.filename,
+ opensearch_client,
+ replace=self.replace_duplicates,
+ owner_user_id=self.user_id,
+ shared=self.shared,
+ )
+ if duplicate_action == "skip":
+ await self._reconcile_shared_owner(file_task.filename)
+ self.mark_duplicate_skipped(upload_task, file_task)
+ return
+
# Rename cleanup: a connector file keeps a stable id across renames,
# but chunks are keyed by filename/content-hash, so a renamed file
# leaves its OLD-name chunks orphaned. Drop chunks for this id whose
@@ -1071,30 +1090,11 @@ class ConnectorFileProcessor(TaskProcessor):
self.user_id,
keep_filenames=get_filename_aliases(file_task.filename),
shared=self.shared,
+ connector_type=connector_type,
)
> 0
)
- if await self.check_filename_exists(file_task.filename, opensearch_client):
- if not self.replace_duplicates:
- await self._reconcile_shared_owner(file_task.filename)
- file_task.status = TaskStatus.SKIPPED
- file_task.error = None
- file_task.result = {
- "status": "skipped",
- "reason": "duplicate_filename",
- "warning": "A file with this name already exists.",
- }
- file_task.updated_at = time.time()
- upload_task.successful_files += 1
- return
- await self.delete_document_by_filename(
- file_task.filename,
- opensearch_client,
- owner_user_id=self.user_id,
- shared=self.shared,
- )
-
# Create temporary file from document content
suffix = os.path.splitext(file_task.filename)[1]
if not suffix:
@@ -1353,6 +1353,7 @@ class S3FileProcessor(TaskProcessor):
owner_email: str = None,
models_service=None,
docling_service=None,
+ replace_duplicates: bool = False,
):
import boto3
@@ -1367,6 +1368,7 @@ class S3FileProcessor(TaskProcessor):
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
+ self.replace_duplicates = replace_duplicates
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
@@ -1378,6 +1380,21 @@ class S3FileProcessor(TaskProcessor):
file_task.updated_at = time.time()
try:
+ # The S3 key doubles as the indexed filename, so the duplicate
+ # gate can run before downloading the object.
+ opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
+ self.owner_user_id, self.jwt_token
+ )
+ duplicate_action = await self.resolve_duplicate_filename(
+ item,
+ opensearch_client,
+ replace=self.replace_duplicates,
+ owner_user_id=self.owner_user_id,
+ )
+ if duplicate_action == "skip":
+ self.mark_duplicate_skipped(upload_task, file_task)
+ return
+
suffix = os.path.splitext(item)[1]
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Download object to temporary file
@@ -1473,33 +1490,15 @@ class LangflowFileProcessor(TaskProcessor):
self.owner_user_id, self.jwt_token
)
- filename_exists = await self.check_filename_exists(original_filename, opensearch_client)
-
- if filename_exists and not self.replace_duplicates:
- # Duplicate exists and user hasn't confirmed replacement
- file_task.status = TaskStatus.FAILED
- file_task.error = f"File with name '{original_filename}' already exists"
- file_task.updated_at = time.time()
- upload_task.failed_files += 1
+ duplicate_action = await self.resolve_duplicate_filename(
+ original_filename,
+ opensearch_client,
+ replace=self.replace_duplicates,
+ owner_user_id=self.owner_user_id,
+ )
+ if duplicate_action == "skip":
+ self.mark_duplicate_skipped(upload_task, file_task)
return
- elif filename_exists and self.replace_duplicates:
- # Delete existing document before uploading new one
- logger.info(f"Replacing existing document: {original_filename}")
- await self.delete_document_by_filename(
- original_filename,
- opensearch_client,
- owner_user_id=self.owner_user_id,
- )
- # Refresh index to make deletion visible before processing.
- # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,
- # so it must run under the admin/service client, not the user client.
- try:
- await clients.opensearch.indices.refresh(index=get_index_name())
- except Exception as refresh_error:
- logger.warning(
- "Failed to refresh index after delete",
- error=str(refresh_error),
- )
# Read file content for processing
with open(item, "rb") as f:
diff --git a/src/services/connector_access_service.py b/src/services/connector_access_service.py
index a662fc1e8..613d5f8aa 100644
--- a/src/services/connector_access_service.py
+++ b/src/services/connector_access_service.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncSession
-from connectors.registry import get_connector_classes
+from connectors.registry import get_connector_class, get_connector_classes
from db.repositories import WorkspaceConfigRepo
CONNECTOR_ACCESS_SECTION = "connector_access"
@@ -13,12 +13,12 @@ CONNECTOR_ACCESS_SECTION = "connector_access"
# without touching this module.
CONNECTOR_TYPES: tuple[str, ...] = tuple(cls.CONNECTOR_TYPE for cls in get_connector_classes())
-_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos", "azure_blob"})
-
def is_bucket_connector_type(connector_type: str) -> bool:
- """True for object-storage "bucket" connectors (S3, IBM COS, Azure Blob)."""
- return connector_type in _BUCKET_CONNECTOR_TYPES
+ """True for object-storage "bucket" connectors, per the class's
+ CONNECTOR_KIND declaration (registry-derived, no hardcoded type list)."""
+ cls = get_connector_class(connector_type)
+ return cls is not None and cls.CONNECTOR_KIND == "bucket"
def governable_connector_types() -> tuple[str, ...]:
@@ -31,7 +31,7 @@ def governable_connector_types() -> tuple[str, ...]:
if not is_azure_blob_enabled():
types = tuple(t for t in types if t != "azure_blob")
if is_cloud_context() and not IBM_AUTH_ENABLED:
- types = tuple(t for t in types if t not in _BUCKET_CONNECTOR_TYPES)
+ types = tuple(t for t in types if not is_bucket_connector_type(t))
return types
diff --git a/src/services/knowledge_filter_service.py b/src/services/knowledge_filter_service.py
index 8937a041c..91cc5334f 100644
--- a/src/services/knowledge_filter_service.py
+++ b/src/services/knowledge_filter_service.py
@@ -34,7 +34,7 @@ class KnowledgeFilterService:
from config.settings import clients, get_index_name
from utils.logging_config import get_logger
- from utils.opensearch_queries import build_existing_filenames_agg_body
+ from utils.opensearch_filenames import find_existing_filenames
data_sources_by_filter: list[list[str] | None] = []
all_filenames = set()
@@ -58,13 +58,9 @@ class KnowledgeFilterService:
if not all_filenames or clients.opensearch is None:
return
- existence_result = await clients.opensearch.search(
- index=get_index_name(),
- body=build_existing_filenames_agg_body(list(all_filenames)),
+ existing_filenames = await find_existing_filenames(
+ all_filenames, clients.opensearch, get_index_name()
)
- existing_filenames = {
- bucket["key"] for bucket in existence_result["aggregations"]["filenames"]["buckets"]
- }
for knowledge_filter, data_sources in zip(filters, data_sources_by_filter, strict=True):
if data_sources:
diff --git a/src/utils/opensearch_filenames.py b/src/utils/opensearch_filenames.py
new file mode 100644
index 000000000..4f3b154a2
--- /dev/null
+++ b/src/utils/opensearch_filenames.py
@@ -0,0 +1,52 @@
+"""Shared filename-existence checks against the documents index.
+
+Duplicate-filename decisions happen at several altitudes — the UI pre-check
+endpoints, the API sync pre-filters, and the per-file processor backstop.
+They must share one query semantic or the layers can disagree about what
+counts as a duplicate. Every caller goes through this module: one terms
+aggregation over exact ``filename`` keywords, with alias expansion via
+``get_filename_aliases`` where a single logical filename is being checked.
+
+The aggregation matters: collecting filenames from search *hits* undercounts
+when a many-chunk document crowds other matches out of the hits window.
+"""
+
+from collections.abc import Iterable
+
+from utils.file_utils import get_filename_aliases
+from utils.opensearch_queries import build_existing_filenames_agg_body
+
+
+async def find_existing_filenames(
+ candidates: Iterable[str],
+ opensearch_client,
+ index: str,
+) -> set[str]:
+ """Return the subset of exact candidate filenames that have at least one
+ indexed chunk visible to this client.
+
+ Callers own alias expansion (pass every variant that should count) and
+ error handling — OpenSearch exceptions propagate.
+ """
+ unique = sorted({c for c in candidates if c})
+ if not unique:
+ return set()
+ response = await opensearch_client.search(
+ index=index,
+ body=build_existing_filenames_agg_body(unique),
+ )
+ buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])
+ return {bucket["key"] for bucket in buckets if bucket.get("key")}
+
+
+async def filename_exists(
+ filename: str,
+ opensearch_client,
+ index: str,
+) -> bool:
+ """True when this filename — or any of its ingestion aliases — has indexed
+ chunks visible to this client."""
+ aliases = get_filename_aliases(filename)
+ if not aliases:
+ return False
+ return bool(await find_existing_filenames(aliases, opensearch_client, index))
diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.py
deleted file mode 100644
index a292277ca..000000000
--- a/tests/unit/api/test_delete_chunks_by_document_ids.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""Unit tests for the bulk-delete helper used by the orphan-reconcile pass.
-
-Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:
-- empty input is a no-op (no OpenSearch call)
-- non-empty input enumerates visible chunk _ids and deletes them one-by-one
- by primary id (DLS-safe; `delete_by_query` is silently filtered under DLS)
-- returns the count of successful single deletes
-"""
-
-import sys
-from pathlib import Path
-from unittest.mock import AsyncMock
-
-import pytest
-
-ROOT = Path(__file__).resolve().parent.parent.parent.parent
-SRC = ROOT / "src"
-if str(SRC) not in sys.path:
- sys.path.insert(0, str(SRC))
-
-
-async def test_empty_ids_short_circuits_without_calling_opensearch():
- from api.documents import delete_chunks_by_document_ids
-
- opensearch_client = AsyncMock()
- deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")
-
- assert deleted == 0
- opensearch_client.search.assert_not_awaited()
- opensearch_client.delete.assert_not_awaited()
- opensearch_client.delete_by_query.assert_not_awaited()
-
-
-async def test_deletes_each_visible_chunk_id_by_primary_id():
- """The helper must enumerate visible chunk _ids and issue a primary-id
- delete for each. delete_by_query is forbidden because DLS / certain
- security plugins silently no-op it (returns deleted:N but leaves docs)."""
- from api.documents import delete_chunks_by_document_ids
-
- opensearch_client = AsyncMock()
- write_client = AsyncMock()
- # Visible chunk _ids for the requested document_ids.
- chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]
- opensearch_client.search.return_value = {
- "_scroll_id": None,
- "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},
- }
- write_client.delete.return_value = {"result": "deleted"}
-
- deleted = await delete_chunks_by_document_ids(
- ["doc-a", "doc-b"],
- opensearch_client,
- "test-index",
- write_opensearch_client=write_client,
- )
-
- assert deleted == len(chunk_ids)
-
- # delete_by_query must NOT be used — silently filtered under DLS.
- opensearch_client.delete_by_query.assert_not_awaited()
- write_client.delete_by_query.assert_not_awaited()
-
- # Search query must target the document_id field with terms(...).
- search_call = opensearch_client.search.await_args
- assert search_call.kwargs["index"] == "test-index"
- assert search_call.kwargs["body"]["query"] == {"terms": {"document_id": ["doc-a", "doc-b"]}}
-
- # One primary-id delete per visible chunk, refresh=True so the delete is
- # immediately visible to the re-index that typically follows.
- opensearch_client.delete.assert_not_awaited()
- delete_calls = write_client.delete.await_args_list
- assert len(delete_calls) == len(chunk_ids)
- for call, cid in zip(delete_calls, chunk_ids, strict=True):
- assert call.kwargs["index"] == "test-index"
- assert call.kwargs["id"] == cid
- assert call.kwargs.get("refresh") is True
-
-
-async def test_returns_zero_when_no_visible_chunks_match():
- """When the search returns no hits, no deletes are issued and the count
- is zero. This is the steady-state path (nothing to clean up)."""
- from api.documents import delete_chunks_by_document_ids
-
- opensearch_client = AsyncMock()
- write_client = AsyncMock()
- opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}
-
- deleted = await delete_chunks_by_document_ids(
- ["abc"],
- opensearch_client,
- "test-index",
- write_opensearch_client=write_client,
- )
-
- assert deleted == 0
- opensearch_client.delete.assert_not_awaited()
- write_client.delete.assert_not_awaited()
-
-
-async def test_custom_field_parameter_is_used_in_query():
- """Passing field='connector_file_id' must query that field and still delete
- through the trusted backend write client."""
- from api.documents import delete_chunks_by_document_ids
-
- opensearch_client = AsyncMock()
- write_client = AsyncMock()
- opensearch_client.search.return_value = {
- "_scroll_id": None,
- "hits": {"hits": [{"_id": "chunk-x"}]},
- }
- write_client.delete.return_value = {"result": "deleted"}
-
- deleted = await delete_chunks_by_document_ids(
- ["src-file-1"],
- opensearch_client,
- "test-index",
- write_opensearch_client=write_client,
- field="connector_file_id",
- )
-
- assert deleted == 1
- search_call = opensearch_client.search.await_args
- assert search_call.kwargs["body"]["query"] == {"terms": {"connector_file_id": ["src-file-1"]}}
- opensearch_client.delete.assert_not_awaited()
- write_client.delete.assert_awaited_once()
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index e0e982af1..3559f95bd 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -6,7 +6,8 @@ The orphan-deletion safety net must:
- preserve files present in any active connection,
- enumerate visible chunks with the user-scoped client, then delete by primary
ID with the trusted backend OpenSearch client,
-- query either `document_id` or `connector_file_id` depending on the ingest path.
+- match BOTH `document_id` and `connector_file_id` so orphan deletion covers
+ chunks from either ingest layout (see connectors.chunk_cleanup).
"""
import json
@@ -30,12 +31,13 @@ def _make_connection(connection_id: str, is_active: bool = True):
def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):
connector = MagicMock()
connector.is_authenticated = authenticated
+ response = {"files": [{"id": fid} for fid in remote_file_ids]}
if raise_on_list:
connector.list_files = AsyncMock(side_effect=RuntimeError("graph 503"))
+ connector.list_selected_files = AsyncMock(side_effect=RuntimeError("graph 503"))
else:
- connector.list_files = AsyncMock(
- return_value={"files": [{"id": fid} for fid in remote_file_ids]}
- )
+ connector.list_files = AsyncMock(return_value=response)
+ connector.list_selected_files = AsyncMock(return_value=response)
return connector
@@ -222,13 +224,75 @@ async def test_happy_path_deletes_orphans(monkeypatch):
opensearch_client.delete.assert_not_awaited()
search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["query"] == {"terms": {"document_id": ["b"]}}
+ shoulds = search_body["query"]["bool"]["filter"][0]["bool"]["should"]
+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}
+ assert fields == {
+ "document_id": ["b"],
+ "connector_file_id": ["b"],
+ "connector_file_id.keyword": ["b"],
+ }
assert [call.kwargs["id"] for call in write_client.delete.await_args_list] == [
"chunk-b-1",
"chunk-b-2",
]
+async def test_happy_path_private_scopes_to_owner(monkeypatch):
+ """Private orphan cleanup must filter by connector_type and owner."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])
+ _patch_write_client(monkeypatch)
+ sm = _make_session_manager(opensearch_client)
+
+ await reconcile_orphans_for_connector_type(
+ connector_type="google_drive",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ filters = search_body["query"]["bool"]["filter"]
+ assert {"term": {"connector_type": "google_drive"}} in filters
+ assert {"term": {"owner": "alice"}} in filters
+
+
+async def test_happy_path_shared_scopes_to_ownerless(monkeypatch):
+ """Shared orphan cleanup must filter by connector_type and ownerless chunks."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])
+ _patch_write_client(monkeypatch)
+ sm = _make_session_manager(opensearch_client)
+
+ await reconcile_orphans_for_connector_type(
+ connector_type="ibm_cos",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ shared=True,
+ )
+
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ filters = search_body["query"]["bool"]["filter"]
+ assert {"term": {"connector_type": "ibm_cos"}} in filters
+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters
+ assert {"term": {"owner": "alice"}} not in filters
+
+
@pytest.mark.asyncio
async def test_delete_failure_does_not_raise(monkeypatch):
from api.connectors import reconcile_orphans_for_connector_type
@@ -310,6 +374,8 @@ async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
@pytest.mark.asyncio
async def test_paginated_listing_aggregates_all_pages():
+ """Connectors without cfg (e.g. bucket connectors) use the paginated
+ list_files() path. Verify that all pages are consumed."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -320,6 +386,7 @@ async def test_paginated_listing_aggregates_all_pages():
{"files": [{"id": "b"}, {"id": "c"}]},
]
connector.list_files = AsyncMock(side_effect=pages)
+ del connector.cfg
service = _make_service([conn], connector_lookup={"c1": connector})
opensearch_client = _make_opensearch_client()
@@ -340,7 +407,11 @@ async def test_paginated_listing_aggregates_all_pages():
@pytest.mark.asyncio
-async def test_connector_file_id_field_used_when_specified(monkeypatch):
+async def test_orphan_delete_matches_both_id_layouts(monkeypatch):
+ """Orphan deletion must cover chunks from either ingest layout: the
+ standard path keys chunks by connector_file_id (document_id is a content
+ hash) while the Langflow path keys them by document_id. A single-field
+ query would miss one of them."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -357,42 +428,21 @@ async def test_connector_file_id_field_used_when_specified(monkeypatch):
session_manager=sm,
jwt_token=None,
existing_file_ids=["sp-guid-a", "sp-guid-b"],
- id_field="connector_file_id",
)
assert result == ["sp-guid-b"]
search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}
+ shoulds = search_body["query"]["bool"]["filter"][0]["bool"]["should"]
+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}
+ assert fields == {
+ "document_id": ["sp-guid-b"],
+ "connector_file_id": ["sp-guid-b"],
+ "connector_file_id.keyword": ["sp-guid-b"],
+ }
assert write_client.delete.await_count == 2
opensearch_client.delete.assert_not_awaited()
-async def test_document_id_field_used_by_default(monkeypatch):
- from api.connectors import reconcile_orphans_for_connector_type
-
- conn = _make_connection("c1")
- connector = _make_connector(remote_file_ids=["lf-id-a"])
- service = _make_service([conn], connector_lookup={"c1": connector})
- opensearch_client = _make_opensearch_client(chunk_ids=["chunk-lf-b-0"])
- write_client = _patch_write_client(monkeypatch)
- sm = _make_session_manager(opensearch_client)
-
- result = await reconcile_orphans_for_connector_type(
- connector_type="sharepoint",
- user_id="alice",
- connector_service=service,
- session_manager=sm,
- jwt_token=None,
- existing_file_ids=["lf-id-a", "lf-id-b"],
- )
-
- assert result == ["lf-id-b"]
- search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}
- write_client.delete.assert_awaited_once()
-
-
@pytest.mark.asyncio
async def test_preview_subtracts_orphans_from_resync_count(monkeypatch):
from api.connectors import _preview_orphans_for_connector_type
diff --git a/tests/unit/api/test_sync_existing_connector_files.py b/tests/unit/api/test_sync_existing_connector_files.py
new file mode 100644
index 000000000..a9718a4f8
--- /dev/null
+++ b/tests/unit/api/test_sync_existing_connector_files.py
@@ -0,0 +1,144 @@
+"""Outcome contract of _sync_existing_connector_files — the shared
+no-selection Sync flow behind both connector_sync and sync_all_connectors."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from api import connectors as connectors_api
+
+
+def _make_services():
+ connector_service = MagicMock()
+ connector_service.sync_specific_files = AsyncMock(return_value="task-1")
+ connector_service.sync_connector_files = AsyncMock(return_value="task-2")
+ connector_service.get_connector = AsyncMock(return_value=MagicMock())
+ working_connection = MagicMock()
+ working_connection.connection_id = "conn-1"
+ return connector_service, working_connection
+
+
+async def _run(
+ connector_service,
+ working_connection,
+ *,
+ connector_type="google_drive",
+ existing_file_ids=None,
+ existing_filenames=None,
+ **kwargs,
+):
+ return await connectors_api._sync_existing_connector_files(
+ connector_type=connector_type,
+ working_connection=working_connection,
+ user_id="user-1",
+ connector_service=connector_service,
+ session_manager=MagicMock(),
+ jwt_token="jwt",
+ existing_file_ids=existing_file_ids or [],
+ existing_filenames=existing_filenames or [],
+ id_field="connector_file_id",
+ **kwargs,
+ )
+
+
+async def test_oauth_sync_uses_connector_replace_capability(monkeypatch):
+ monkeypatch.setattr(
+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])
+ )
+ connector_service, working_connection = _make_services()
+
+ result = await _run(connector_service, working_connection, existing_file_ids=["f1", "f2"])
+
+ assert result == {"outcome": "synced", "task_id": "task-1"}
+ call = connector_service.sync_specific_files.await_args
+ assert call.args[2] == ["f1", "f2"]
+ # google_drive declares SYNC_REPLACES_DUPLICATES=True.
+ assert call.kwargs["replace_duplicates"] is True
+
+
+async def test_deleted_only_when_orphan_reconcile_removes_everything(monkeypatch):
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=["f1", "f2"]),
+ )
+ connector_service, working_connection = _make_services()
+
+ result = await _run(connector_service, working_connection, existing_file_ids=["f1", "f2"])
+
+ assert result == {"outcome": "deleted_only"}
+ connector_service.sync_specific_files.assert_not_awaited()
+
+
+async def test_reconcile_false_skips_orphan_pass(monkeypatch):
+ reconcile = AsyncMock(return_value=["f1"])
+ monkeypatch.setattr(connectors_api, "reconcile_orphans_for_connector_type", reconcile)
+ connector_service, working_connection = _make_services()
+
+ result = await _run(
+ connector_service,
+ working_connection,
+ existing_file_ids=["f1"],
+ reconcile=False,
+ )
+
+ assert result["outcome"] == "synced"
+ reconcile.assert_not_awaited()
+
+
+async def test_timestamp_connector_up_to_date_when_no_changes(monkeypatch):
+ monkeypatch.setattr(
+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])
+ )
+ monkeypatch.setattr(connectors_api, "bucket_changed_file_ids", AsyncMock(return_value=[]))
+ connector_service, working_connection = _make_services()
+
+ result = await _run(
+ connector_service,
+ working_connection,
+ connector_type="aws_s3",
+ existing_file_ids=["k1"],
+ )
+
+ assert result == {"outcome": "up_to_date"}
+ connector_service.sync_specific_files.assert_not_awaited()
+
+
+async def test_timestamp_connector_syncs_only_changed_ids_with_replace(monkeypatch):
+ monkeypatch.setattr(
+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])
+ )
+ monkeypatch.setattr(connectors_api, "bucket_changed_file_ids", AsyncMock(return_value=["k2"]))
+ connector_service, working_connection = _make_services()
+
+ result = await _run(
+ connector_service,
+ working_connection,
+ connector_type="aws_s3",
+ existing_file_ids=["k1", "k2"],
+ )
+
+ assert result == {"outcome": "synced", "task_id": "task-1"}
+ call = connector_service.sync_specific_files.await_args
+ assert call.args[2] == ["k2"]
+ assert call.kwargs["replace_duplicates"] is True
+
+
+async def test_filename_fallback_when_no_ids(monkeypatch):
+ connector_service, working_connection = _make_services()
+
+ result = await _run(
+ connector_service,
+ working_connection,
+ existing_filenames=["a.pdf", "b.pdf"],
+ )
+
+ assert result == {"outcome": "synced", "task_id": "task-2"}
+ call = connector_service.sync_connector_files.await_args
+ assert call.kwargs["filename_filter"] == {"a.pdf", "b.pdf"}
diff --git a/tests/unit/connectors/test_change_detection_capability.py b/tests/unit/connectors/test_change_detection_capability.py
new file mode 100644
index 000000000..3e61d23e7
--- /dev/null
+++ b/tests/unit/connectors/test_change_detection_capability.py
@@ -0,0 +1,49 @@
+"""Change-detection strategy is a connector capability, not a hardcoded list.
+
+``_connector_uses_timestamp_change_detection`` reads ``CHANGE_DETECTION`` from
+the registered connector class, and ``is_bucket_connector_type`` derives from
+``CONNECTOR_KIND`` — neither consults a type-string set anymore.
+"""
+
+import pytest
+
+from api.connectors import _connector_uses_timestamp_change_detection
+from connectors.aws_s3 import S3Connector
+from connectors.base import BaseConnector
+from services.connector_access_service import is_bucket_connector_type
+
+
+def test_base_default_is_replace_always():
+ assert BaseConnector.CHANGE_DETECTION == "replace_always"
+
+
+def test_s3_declares_timestamp_change_detection():
+ assert S3Connector.CHANGE_DETECTION == "timestamp"
+ assert _connector_uses_timestamp_change_detection("aws_s3") is True
+
+
[email protected]("connector_type", ["google_drive", "sharepoint", "onedrive"])
+def test_oauth_connectors_use_replace_always(connector_type):
+ assert _connector_uses_timestamp_change_detection(connector_type) is False
+
+
+def test_unknown_type_defaults_to_replace_always():
+ assert _connector_uses_timestamp_change_detection("no_such_connector") is False
+
+
+def test_capability_read_from_registered_class(monkeypatch):
+ class TimestampConnector:
+ CONNECTOR_TYPE = "custom_ts"
+ CHANGE_DETECTION = "timestamp"
+
+ monkeypatch.setattr(
+ "connectors.registry.get_connector_class",
+ lambda connector_type: TimestampConnector if connector_type == "custom_ts" else None,
+ )
+ assert _connector_uses_timestamp_change_detection("custom_ts") is True
+
+
+def test_is_bucket_connector_type_derived_from_kind():
+ assert is_bucket_connector_type("aws_s3") is True
+ assert is_bucket_connector_type("google_drive") is False
+ assert is_bucket_connector_type("no_such_connector") is False
diff --git a/tests/unit/connectors/test_chunk_cleanup.py b/tests/unit/connectors/test_chunk_cleanup.py
new file mode 100644
index 000000000..ff7370a05
--- /dev/null
+++ b/tests/unit/connectors/test_chunk_cleanup.py
@@ -0,0 +1,165 @@
+"""Contract of connectors.chunk_cleanup — the single deletion path for a
+connector file's indexed chunks (orphan reconcile, 404 cleanup, rename
+cleanup all go through it).
+
+- empty input is a no-op (no OpenSearch call)
+- the query matches BOTH id layouts (document_id and connector_file_id)
+- DLS-safe: visible chunk _ids are enumerated with the caller's client, then
+ deleted one-by-one by primary id with the trusted backend write client
+ (`delete_by_query` is silently filtered under DLS)
+- owner/shared scoping and rename keep_filenames shape the query
+"""
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from connectors.chunk_cleanup import (
+ build_connector_file_chunks_query,
+ delete_connector_file_chunks,
+)
+
+
+def write_client(monkeypatch):
+ import config.settings as cfg
+
+ client = AsyncMock()
+ client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(cfg.clients, "opensearch", client)
+ return client
+
+
+def _id_terms(query) -> dict:
+ shoulds = query["bool"]["filter"][0]["bool"]["should"]
+ return {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}
+
+
+def test_query_matches_both_id_fields():
+ query = build_connector_file_chunks_query(["file-1", "file-2"])
+ fields = _id_terms(query)
+ assert fields == {
+ "document_id": ["file-1", "file-2"],
+ "connector_file_id": ["file-1", "file-2"],
+ "connector_file_id.keyword": ["file-1", "file-2"],
+ }
+ assert query["bool"]["filter"][0]["bool"]["minimum_should_match"] == 1
+
+
+def test_query_owner_scoping():
+ query = build_connector_file_chunks_query(["f"], owner_user_id="alice")
+ assert {"term": {"owner": "alice"}} in query["bool"]["filter"]
+
+
+def test_query_shared_targets_ownerless_chunks():
+ query = build_connector_file_chunks_query(["f"], owner_user_id="alice", shared=True)
+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in query["bool"]["filter"]
+ assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]
+
+
+def test_query_keep_filenames_excluded():
+ query = build_connector_file_chunks_query(["f"], keep_filenames=["new.pdf", "new_.pdf"])
+ assert query["bool"]["must_not"] == [{"terms": {"filename": ["new.pdf", "new_.pdf"]}}]
+
+
+def test_query_unscoped_when_no_owner():
+ query = build_connector_file_chunks_query(["f"])
+ assert len(query["bool"]["filter"]) == 1
+ assert "must_not" not in query["bool"]
+
+
+def test_query_connector_type_filter():
+ query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos")
+ assert {"term": {"connector_type": "ibm_cos"}} in query["bool"]["filter"]
+
+
+def test_query_connector_type_with_private_owner():
+ query = build_connector_file_chunks_query(
+ ["f"], connector_type="google_drive", owner_user_id="alice"
+ )
+ filters = query["bool"]["filter"]
+ assert {"term": {"connector_type": "google_drive"}} in filters
+ assert {"term": {"owner": "alice"}} in filters
+
+
+def test_query_connector_type_with_shared_scope():
+ query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos", shared=True)
+ filters = query["bool"]["filter"]
+ assert {"term": {"connector_type": "ibm_cos"}} in filters
+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters
+ assert {"term": {"owner": "alice"}} not in filters
+
+
+def test_query_no_connector_type_when_none():
+ query = build_connector_file_chunks_query(["f"])
+ for f in query["bool"]["filter"]:
+ if "term" in f:
+ assert "connector_type" not in f["term"]
+
+
+async def test_empty_ids_short_circuit_without_calling_opensearch(write_client):
+ opensearch_client = AsyncMock()
+
+ deleted = await delete_connector_file_chunks([], opensearch_client)
+
+ assert deleted == 0
+ opensearch_client.search.assert_not_awaited()
+ write_client.delete.assert_not_awaited()
+
+
+async def test_deletes_each_visible_chunk_id_by_primary_id(write_client, monkeypatch):
+ """Visible chunk _ids are enumerated with the caller's client and deleted
+ one-by-one via the backend write client. delete_by_query is forbidden —
+ DLS / certain security plugins silently no-op it."""
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")
+
+ opensearch_client = AsyncMock()
+ chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]
+ opensearch_client.search.return_value = {
+ "_scroll_id": None,
+ "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},
+ }
+
+ deleted = await delete_connector_file_chunks(
+ ["doc-a", "doc-b"], opensearch_client, refresh=True
+ )
+
+ assert deleted == len(chunk_ids)
+ opensearch_client.delete_by_query.assert_not_awaited()
+ write_client.delete_by_query.assert_not_awaited()
+
+ search_call = opensearch_client.search.await_args
+ assert search_call.kwargs["index"] == "test-index"
+ fields = _id_terms(search_call.kwargs["body"]["query"])
+ assert fields["document_id"] == ["doc-a", "doc-b"]
+ assert fields["connector_file_id"] == ["doc-a", "doc-b"]
+ assert fields["connector_file_id.keyword"] == ["doc-a", "doc-b"]
+
+ opensearch_client.delete.assert_not_awaited()
+ delete_calls = write_client.delete.await_args_list
+ assert [call.kwargs["id"] for call in delete_calls] == chunk_ids
+ for call in delete_calls:
+ assert call.kwargs["index"] == "test-index"
+ assert call.kwargs.get("refresh") is True
+
+
+async def test_returns_zero_when_no_visible_chunks_match(write_client):
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}
+
+ deleted = await delete_connector_file_chunks(["abc"], opensearch_client)
+
+ assert deleted == 0
+ write_client.delete.assert_not_awaited()
+
+
+async def test_raises_when_write_client_unavailable(monkeypatch):
+ import config.settings as cfg
+
+ monkeypatch.setattr(cfg.clients, "opensearch", None)
+ with pytest.raises(RuntimeError, match="write client"):
+ await delete_connector_file_chunks(["abc"], AsyncMock())
diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.py
index 8d0d98246..fc74f3632 100644
--- a/tests/unit/connectors/test_connector_file_type_validation.py
+++ b/tests/unit/connectors/test_connector_file_type_validation.py
@@ -27,14 +27,14 @@ async def test_sync_specific_files_does_not_raise_on_incompatible_type():
connector.is_authenticated = True
# Mock list_files returning an incompatible file (e.g. an .exe)
- connector.list_files = AsyncMock(
- return_value={
- "files": [
- {"id": "file-1", "name": "document.pdf"},
- {"id": "file-2", "name": "program.exe"},
- ]
- }
- )
+ expanded_response = {
+ "files": [
+ {"id": "file-1", "name": "document.pdf"},
+ {"id": "file-2", "name": "program.exe"},
+ ]
+ }
+ connector.list_files = AsyncMock(return_value=expanded_response)
+ connector.list_selected_files = AsyncMock(return_value=expanded_response)
connector.cfg = MagicMock()
service.get_connector = AsyncMock(return_value=connector)
@@ -104,18 +104,18 @@ async def test_connector_check_duplicates():
connector.authenticate = AsyncMock(return_value=True)
# Mock folder expansion
- connector.list_files = AsyncMock(
- return_value={
- "files": [
- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
- {
- "id": "file-2",
- "name": "new_file.docx",
- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- },
- ]
- }
- )
+ expanded_files = {
+ "files": [
+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
+ {
+ "id": "file-2",
+ "name": "new_file.docx",
+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ },
+ ]
+ }
+ connector.list_files = AsyncMock(return_value=expanded_files)
+ connector.list_selected_files = AsyncMock(return_value=expanded_files)
connector.cfg = MagicMock()
connector_service.get_connector = AsyncMock(return_value=connector)
@@ -126,7 +126,9 @@ async def test_connector_check_duplicates():
# Mock search return value: existing.pdf exists, new_file.docx does not
opensearch_client.search = AsyncMock(
- return_value={"hits": {"hits": [{"_source": {"filename": "existing.pdf"}}]}}
+ return_value={
+ "aggregations": {"filenames": {"buckets": [{"key": "existing.pdf", "doc_count": 1}]}}
+ }
)
user = MagicMock()
@@ -286,18 +288,18 @@ async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selected
connector.is_authenticated = True
connector.authenticate = AsyncMock(return_value=True)
connector.cfg = MagicMock()
- connector.list_files = AsyncMock(
- return_value={
- "files": [
- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
- {
- "id": "file-2",
- "name": "existing.docx",
- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- },
- ]
- }
- )
+ expanded_files = {
+ "files": [
+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
+ {
+ "id": "file-2",
+ "name": "existing.docx",
+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ },
+ ]
+ }
+ connector.list_files = AsyncMock(return_value=expanded_files)
+ connector.list_selected_files = AsyncMock(return_value=expanded_files)
connector_service.get_connector = AsyncMock(return_value=connector)
connector_service.sync_specific_files = AsyncMock(return_value="task-id")
@@ -306,11 +308,13 @@ async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selected
session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
opensearch_client.search = AsyncMock(
return_value={
- "hits": {
- "hits": [
- {"_source": {"filename": "existing.pdf"}},
- {"_source": {"filename": "existing.docx"}},
- ]
+ "aggregations": {
+ "filenames": {
+ "buckets": [
+ {"key": "existing.pdf", "doc_count": 1},
+ {"key": "existing.docx", "doc_count": 1},
+ ]
+ }
}
}
)
@@ -356,18 +360,18 @@ async def test_connector_sync_skip_duplicates_submits_only_expanded_non_duplicat
connector.is_authenticated = True
connector.authenticate = AsyncMock(return_value=True)
connector.cfg = MagicMock()
- connector.list_files = AsyncMock(
- return_value={
- "files": [
- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
- {
- "id": "file-2",
- "name": "new_file.docx",
- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- },
- ]
- }
- )
+ expanded_files = {
+ "files": [
+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},
+ {
+ "id": "file-2",
+ "name": "new_file.docx",
+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ },
+ ]
+ }
+ connector.list_files = AsyncMock(return_value=expanded_files)
+ connector.list_selected_files = AsyncMock(return_value=expanded_files)
connector_service.get_connector = AsyncMock(return_value=connector)
connector_service.sync_specific_files = AsyncMock(return_value="task-id")
@@ -375,7 +379,9 @@ async def test_connector_sync_skip_duplicates_submits_only_expanded_non_duplicat
opensearch_client = AsyncMock()
session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
opensearch_client.search = AsyncMock(
- return_value={"hits": {"hits": [{"_source": {"filename": "existing.pdf"}}]}}
+ return_value={
+ "aggregations": {"filenames": {"buckets": [{"key": "existing.pdf", "doc_count": 1}]}}
+ }
)
response = await connectors_api.connector_sync(
diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.py
deleted file mode 100644
index 4ecc23d53..000000000
--- a/tests/unit/connectors/test_langflow_connector_txt_to_md.py
+++ /dev/null
@@ -1,115 +0,0 @@
-"""Regression guard for SharePoint .txt ingestion via the Langflow connector
-path.
-
-Background: Langflow's docling component fails on text/plain. The
-user-upload path already applies a .txt -> .md rename (commit f6b9fe0).
-This test pins that the connector path
-(`ConnectorService.process_connector_document`) applies the same rule, so
-a SharePoint .txt reaches Langflow as .md / text/markdown — not
-the unhandled text/plain shape that previously broke ingestion.
-
-This test drives the production code path; if the rename is ever removed
-or shorted around, the assertions will catch it.
-"""
-
-import sys
-from datetime import datetime
-from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-ROOT = Path(__file__).resolve().parent.parent.parent.parent
-SRC = ROOT / "src"
-if str(SRC) not in sys.path:
- sys.path.insert(0, str(SRC))
-
-
-def _make_service():
- """Build a ConnectorService with the surface
- process_connector_document touches stubbed.
- """
- from connectors.service import ConnectorService
-
- service = ConnectorService.__new__(ConnectorService)
-
- # Mock Langflow service
- langflow_service = MagicMock()
- langflow_service.upload_and_ingest_file = AsyncMock(return_value={"status": "ok"})
- langflow_service.merge_ui_ingest_settings_into_tweaks = MagicMock(return_value={})
- service.langflow_service = langflow_service
- service.task_service = None
- # _get_effective_sync_jwt passes jwt_token through when no session manager
- service.session_manager = None
-
- return service, langflow_service
-
-
-def _make_document(filename: str, mimetype: str, content: bytes = b"hello world"):
- from connectors.base import ConnectorDocument, DocumentACL
-
- return ConnectorDocument(
- id="graph-item-id-stable",
- filename=filename,
- mimetype=mimetype,
- content=content,
- source_url="https://contoso.sharepoint.com/.../notes.txt",
- acl=DocumentACL(owner="alice"),
- modified_time=datetime(2026, 5, 21),
- created_time=datetime(2026, 5, 1),
- metadata={"site": "marketing"},
- )
-
-
-async def test_sharepoint_txt_is_uploaded_to_langflow_as_md():
- """A connector document with mimetype text/plain must reach Langflow's
- upload_and_ingest_file with a .md filename and text/markdown mimetype."""
- service, langflow_service = _make_service()
- document = _make_document(filename="notes.txt", mimetype="text/plain")
-
- await service.process_connector_document(
- document=document,
- owner_user_id="alice",
- connector_type="sharepoint",
- jwt_token="jwt",
- )
-
- langflow_service.upload_and_ingest_file.assert_awaited_once()
- kwargs = langflow_service.upload_and_ingest_file.await_args.kwargs
- file_tuple = kwargs.get("file_tuple")
- filename, content, mimetype = file_tuple
-
- assert filename.endswith(".md"), (
- f"SharePoint .txt must reach Langflow as .md; got {filename!r}. "
- "Langflow's docling component fails on text/plain — see "
- "langflow_safe_filename_and_mimetype."
- )
- assert mimetype == "text/markdown", (
- f"SharePoint .txt must reach Langflow as text/markdown; got {mimetype!r}."
- )
- # Content bytes are unchanged by the rename.
- assert content == document.content
-
-
-async def test_sharepoint_pdf_passes_through_untouched():
- """The rename rule must NOT touch non-.txt files."""
- service, langflow_service = _make_service()
- document = _make_document(
- filename="report.pdf", mimetype="application/pdf", content=b"%PDF-1.4..."
- )
-
- await service.process_connector_document(
- document=document,
- owner_user_id="alice",
- connector_type="sharepoint",
- jwt_token="jwt",
- )
-
- langflow_service.upload_and_ingest_file.assert_awaited_once()
- kwargs = langflow_service.upload_and_ingest_file.await_args.kwargs
- file_tuple = kwargs.get("file_tuple")
- filename, _content, mimetype = file_tuple
- assert filename == "report.pdf"
- assert mimetype == "application/pdf"
diff --git a/tests/unit/connectors/test_list_selected_files.py b/tests/unit/connectors/test_list_selected_files.py
new file mode 100644
index 000000000..f74e0501b
--- /dev/null
+++ b/tests/unit/connectors/test_list_selected_files.py
@@ -0,0 +1,115 @@
+"""BaseConnector.list_selected_files() — owns the cfg save/restore so
+callers don't have to manually back up and restore file_ids/folder_ids."""
+
+from dataclasses import dataclass
+
+import pytest
+
+from connectors.base import BaseConnector
+
+
+@dataclass
+class _FakeCfg:
+ file_ids: list[str] | None = None
+ folder_ids: list[str] | None = None
+
+
+class _FakeConnector(BaseConnector):
+ CONNECTOR_TYPE = "fake"
+
+ def __init__(self):
+ self.cfg = _FakeCfg(file_ids=["orig-1"], folder_ids=["folder-orig"])
+
+ async def authenticate(self):
+ return True
+
+ async def setup_subscription(self):
+ return ""
+
+ async def list_files(self, page_token=None, max_files=None, **kwargs):
+ return {"files": [{"id": fid} for fid in (self.cfg.file_ids or [])]}
+
+ async def get_file_content(self, file_id):
+ raise NotImplementedError
+
+ async def handle_webhook(self, payload):
+ return []
+
+ async def cleanup_subscription(self, subscription_id):
+ return True
+
+
+async def test_restores_cfg_after_success():
+ conn = _FakeConnector()
+ result = await conn.list_selected_files(["a", "b"])
+ assert [f["id"] for f in result["files"]] == ["a", "b"]
+ assert conn.cfg.file_ids == ["orig-1"]
+ assert conn.cfg.folder_ids == ["folder-orig"]
+
+
+async def test_restores_cfg_after_exception():
+ class _Broken(_FakeConnector):
+ async def list_files(self, **kwargs):
+ raise RuntimeError("boom")
+
+ conn = _Broken()
+ with pytest.raises(RuntimeError, match="boom"):
+ await conn.list_selected_files(["x"])
+ assert conn.cfg.file_ids == ["orig-1"]
+ assert conn.cfg.folder_ids == ["folder-orig"]
+
+
+async def test_folder_ids_cleared_by_default():
+ """list_selected_files sets folder_ids to None during the call."""
+ observed_folder_ids = None
+
+ class _Capture(_FakeConnector):
+ async def list_files(self, **kwargs):
+ nonlocal observed_folder_ids
+ observed_folder_ids = self.cfg.folder_ids
+ return {"files": []}
+
+ conn = _Capture()
+ await conn.list_selected_files(["a"])
+ assert observed_folder_ids is None
+
+
+async def test_raises_when_cfg_is_none():
+ """cfg-less (bucket) connectors must not silently fall back to list_files().
+
+ BaseConnector declares cfg=None as a class default, so a bucket connector
+ reaches list_selected_files with cfg None. Falling back to list_files()
+ there would list the connector's entire account and ignore file_ids —
+ exactly the "all buckets ingested" regression. Callers must gate on
+ `cfg is not None` instead, so this path is a loud error, not a silent
+ whole-account listing.
+ """
+
+ class _NoCfg(_FakeConnector):
+ def __init__(self):
+ self.cfg = None
+
+ conn = _NoCfg()
+ with pytest.raises(NotImplementedError):
+ await conn.list_selected_files(["a"])
+
+
+async def test_folder_ids_passed_through():
+ observed = {}
+
+ class _Capture(_FakeConnector):
+ async def list_files(self, **kwargs):
+ nonlocal observed
+ observed = {"file_ids": self.cfg.file_ids, "folder_ids": self.cfg.folder_ids}
+ return {"files": []}
+
+ conn = _Capture()
+ await conn.list_selected_files(["f1"], folder_ids=["fold-1"])
+ assert observed == {"file_ids": ["f1"], "folder_ids": ["fold-1"]}
+ assert conn.cfg.file_ids == ["orig-1"]
+ assert conn.cfg.folder_ids == ["folder-orig"]
diff --git a/tests/unit/connectors/test_sync_replace_capability.py b/tests/unit/connectors/test_sync_replace_capability.py
new file mode 100644
index 000000000..162e439e5
--- /dev/null
+++ b/tests/unit/connectors/test_sync_replace_capability.py
@@ -0,0 +1,57 @@
+"""Sync replace-on-duplicate is a connector capability, not a hardcoded list.
+
+``_connector_sync_should_replace`` reads ``SYNC_REPLACES_DUPLICATES`` from the
+connector class registered for the type. The BaseConnector default is True so
+a new connector gets change-propagating sync without touching the API module.
+"""
+
+import pytest
+
+from api.connectors import _connector_sync_should_replace
+from connectors.base import BaseConnector
+
+
[email protected]("connector_type", ["google_drive", "sharepoint", "onedrive"])
+def test_builtin_oauth_connectors_replace_on_sync(connector_type):
+ assert _connector_sync_should_replace(connector_type) is True
+
+
+def test_unknown_connector_type_does_not_replace():
+ assert _connector_sync_should_replace("no_such_connector") is False
+
+
+def test_new_connector_inherits_replace_default():
+ class NewConnector(BaseConnector):
+ CONNECTOR_TYPE = "new_connector"
+
+ async def authenticate(self): # pragma: no cover - capability test only
+ return True
+
+ async def setup_subscription(self): # pragma: no cover
+ raise NotImplementedError
+
+ async def list_files(self, page_token=None, max_files=None, **kwargs): # pragma: no cover
+ raise NotImplementedError
+
+ async def get_file_content(self, file_id): # pragma: no cover
+ raise NotImplementedError
+
+ async def handle_webhook(self, payload): # pragma: no cover
+ raise NotImplementedError
+
+ async def cleanup_subscription(self, subscription_id): # pragma: no cover
+ raise NotImplementedError
+
+ assert NewConnector.SYNC_REPLACES_DUPLICATES is True
+
+
+def test_connector_can_opt_out_via_attribute(monkeypatch):
+ class OptOutConnector:
+ CONNECTOR_TYPE = "opt_out"
+ SYNC_REPLACES_DUPLICATES = False
+
+ monkeypatch.setattr(
+ "connectors.registry.get_connector_class",
+ lambda connector_type: OptOutConnector if connector_type == "opt_out" else None,
+ )
+ assert _connector_sync_should_replace("opt_out") is False
diff --git a/tests/unit/connectors/test_sync_specific_files_deleted_ids.py b/tests/unit/connectors/test_sync_specific_files_deleted_ids.py
index 29936d45d..7be77eaab 100644
--- a/tests/unit/connectors/test_sync_specific_files_deleted_ids.py
+++ b/tests/unit/connectors/test_sync_specific_files_deleted_ids.py
@@ -35,7 +35,9 @@ def _make_service(live_files: list[dict]):
connector = MagicMock()
connector.is_authenticated = True
connector.cfg = MagicMock()
- connector.list_files = AsyncMock(return_value={"files": live_files})
+ response = {"files": live_files}
+ connector.list_files = AsyncMock(return_value=response)
+ connector.list_selected_files = AsyncMock(return_value=response)
service.get_connector = AsyncMock(return_value=connector)
return service
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
index a40bffd8a..84423de08 100644
--- a/tests/unit/test_connector_processor_filename_dedupe.py
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -49,6 +49,13 @@ def _make_search_response(has_hit: bool) -> dict:
return {"hits": {"hits": [{"_id": "x"}] if has_hit else []}}
+def _make_filename_agg_response(has_hit: bool, filename: str = "My Report.pdf") -> dict:
+ """The filename duplicate check runs a terms aggregation (see
+ utils/opensearch_filenames), not a hits query."""
+ buckets = [{"key": filename, "doc_count": 1}] if has_hit else []
+ return {"aggregations": {"filenames": {"buckets": buckets}}}
+
+
def _make_upload_task() -> UploadTask:
return UploadTask(task_id="task-1", total_files=1)
@@ -94,6 +101,10 @@ def _wire_connector_processor(
return _make_search_response(rename_stale_exists)
if "document_id" in query_str:
return _make_search_response(hash_exists)
+ if isinstance(body, dict) and "aggs" in body:
+ # Bulk duplicate check (terms aggregation over filename aliases).
+ return _make_filename_agg_response(filename_exists)
+ # Hits-based filename scroll (delete_document_by_filename id collection).
return _make_search_response(filename_exists)
opensearch_client.search = mock_search
@@ -220,9 +231,14 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(
search_call = opensearch_client.search.await_args
query = search_call.kwargs["body"]["query"]
shoulds = query["bool"]["filter"][0]["bool"]["should"]
- fields = {next(iter(c["term"])): next(iter(c["term"].values())) for c in shoulds}
- assert fields["connector_file_id"] == "file-id-1"
- assert fields["document_id"] == "file-id-1"
+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}
+ assert fields["connector_file_id"] == ["file-id-1"]
+ assert fields["document_id"] == ["file-id-1"]
+
+ # ...and stay scoped to this connector type, so an id that collides with a
+ # different connector's id can't take its chunks down with it.
+ terms = [f["term"] for f in query["bool"]["filter"] if "term" in f]
+ assert {"connector_type": "sharepoint"} in terms
@pytest.mark.asyncio
@@ -321,6 +337,10 @@ def _wire_langflow_processor(
return _make_search_response(connector_id_exists)
if document_id:
return _make_search_response(hash_exists)
+ if isinstance(body, dict) and "aggs" in body:
+ # Bulk duplicate check (terms aggregation over filename aliases).
+ return _make_filename_agg_response(filename_exists)
+ # Hits-based filename scroll (delete_document_by_filename id collection).
return _make_search_response(filename_exists)
opensearch_client.search = mock_search
@@ -581,10 +601,14 @@ async def test_rename_cleanup_matches_both_id_fields(monkeypatch, backend_write_
)
shoulds = captured["query"]["bool"]["filter"][0]["bool"]["should"]
- fields = {next(iter(c["term"])) for c in shoulds}
+ fields = {next(iter(c["terms"])) for c in shoulds}
assert fields == {"document_id", "connector_file_id", "connector_file_id.keyword"}
excluded = captured["query"]["bool"]["must_not"][0]["terms"]["filename"]
assert set(get_filename_aliases("Renamed.pdf")).issubset(set(excluded))
+ # Scoped to this connector type, matching the value the chunks were indexed
+ # under, so a colliding id from another connector is left alone.
+ terms = [f["term"] for f in captured["query"]["bool"]["filter"] if "term" in f]
+ assert {"connector_type": "sharepoint"} in terms
@pytest.mark.asyncio
diff --git a/tests/unit/test_connector_sync_bucket_filter_shared.py b/tests/unit/test_connector_sync_bucket_filter_shared.py
index 5307edbed..cc622e63b 100644
--- a/tests/unit/test_connector_sync_bucket_filter_shared.py
+++ b/tests/unit/test_connector_sync_bucket_filter_shared.py
@@ -54,6 +54,10 @@ async def test_bucket_filter_sync_forwards_shared_to_new_and_changed_batches():
user.user_id = "user-1"
with (
+ patch(
+ "api.connectors.has_effective_permission",
+ new=AsyncMock(return_value=True),
+ ),
patch(
"api.connectors.get_synced_file_ids_for_connector",
new=AsyncMock(return_value=(["changed-file-1"], [], "connector_file_id")),
diff --git a/tests/unit/test_opensearch_filenames.py b/tests/unit/test_opensearch_filenames.py
new file mode 100644
index 000000000..b360a80fc
--- /dev/null
+++ b/tests/unit/test_opensearch_filenames.py
@@ -0,0 +1,74 @@
+"""Shared filename-existence executor (utils/opensearch_filenames).
+
+All duplicate-detection altitudes (UI pre-check endpoint, connector duplicate
+classifier, processor backstop) go through these two functions, so their
+query semantics cannot drift apart.
+"""
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from utils.opensearch_filenames import filename_exists, find_existing_filenames
+
+
+def _agg_response(*filenames: str) -> dict:
+ return {
+ "aggregations": {"filenames": {"buckets": [{"key": f, "doc_count": 1} for f in filenames]}}
+ }
+
+
+async def test_find_existing_filenames_uses_terms_aggregation():
+ client = AsyncMock()
+ client.search.return_value = _agg_response("a.pdf")
+
+ result = await find_existing_filenames(["a.pdf", "b.pdf"], client, "docs")
+
+ assert result == {"a.pdf"}
+ body = client.search.await_args.kwargs["body"]
+ # Aggregation, not hits: exact results regardless of per-file chunk counts.
+ assert body["size"] == 0
+ assert set(body["query"]["terms"]["filename"]) == {"a.pdf", "b.pdf"}
+ assert "aggs" in body
+
+
+async def test_find_existing_filenames_dedupes_and_drops_empty():
+ client = AsyncMock()
+ client.search.return_value = _agg_response()
+
+ await find_existing_filenames(["a.pdf", "a.pdf", "", None], client, "docs")
+
+ assert client.search.await_args.kwargs["body"]["query"]["terms"]["filename"] == ["a.pdf"]
+
+
+async def test_find_existing_filenames_empty_input_short_circuits():
+ client = AsyncMock()
+ assert await find_existing_filenames([], client, "docs") == set()
+ client.search.assert_not_awaited()
+
+
+async def test_filename_exists_expands_aliases():
+ client = AsyncMock()
+ client.search.return_value = _agg_response("report.md")
+
+ assert await filename_exists("report.txt", client, "docs") is True
+ queried = set(client.search.await_args.kwargs["body"]["query"]["terms"]["filename"])
+ assert {"report.txt", "report.md"}.issubset(queried)
+
+
+async def test_filename_exists_false_when_no_alias_indexed():
+ client = AsyncMock()
+ client.search.return_value = _agg_response()
+ assert await filename_exists("report.pdf", client, "docs") is False
+
+
+async def test_filename_exists_blank_input_short_circuits():
+ client = AsyncMock()
+ assert await filename_exists("", client, "docs") is False
+ client.search.assert_not_awaited()
diff --git a/tests/unit/test_processor_refresh_uses_admin_client.py b/tests/unit/test_processor_refresh_uses_admin_client.py
index ad1be9cec..bfdcf21f6 100644
--- a/tests/unit/test_processor_refresh_uses_admin_client.py
+++ b/tests/unit/test_processor_refresh_uses_admin_client.py
@@ -54,7 +54,7 @@ async def test_document_processor_refresh_uses_admin_client(monkeypatch, tmp_pat
session_manager=session_manager,
)
processor.check_filename_exists = AsyncMock(return_value=True)
- processor.delete_document_by_filename = AsyncMock()
+ processor.delete_document_by_filename = AsyncMock(return_value=1)
processor.process_document_standard = AsyncMock(
return_value={"status": "indexed", "id": "hash-1"}
)
@@ -98,7 +98,7 @@ async def test_langflow_processor_refresh_uses_admin_client(monkeypatch, tmp_pat
replace_duplicates=True,
)
processor.check_filename_exists = AsyncMock(return_value=True)
- processor.delete_document_by_filename = AsyncMock()
+ processor.delete_document_by_filename = AsyncMock(return_value=1)
# The Langflow path reads the file off disk after the refresh.
item = tmp_path / "report.pdf"
diff --git a/tests/unit/test_processors_filename_alias_retry.py b/tests/unit/test_processors_filename_alias_retry.py
index 7a690a4ef..45e0d7c55 100644
--- a/tests/unit/test_processors_filename_alias_retry.py
+++ b/tests/unit/test_processors_filename_alias_retry.py
@@ -1,4 +1,6 @@
-import asyncio
+"""check_filename_exists issues ONE bulk aliases query (shared semantics) and
+retries it on transient failures."""
+
from unittest.mock import AsyncMock
import pytest
@@ -7,31 +9,40 @@ from models.processors import TaskProcessor
@pytest.mark.asyncio
-async def test_check_filename_exists_does_not_requery_successfully_checked_aliases_on_retry():
- """Once an alias is checked successfully, retries should continue from pending aliases."""
+async def test_check_filename_exists_bulk_queries_all_aliases_and_retries():
+ """All filename aliases go into a single terms-aggregation query; a
+ transient failure retries the same bulk query."""
processor = TaskProcessor()
opensearch_client = AsyncMock()
+ calls = []
+
async def _search_side_effect(*, index, body):
- candidate = body["query"]["term"]["filename"]
- if candidate == "report.md" and _search_side_effect.md_calls == 0:
- _search_side_effect.md_calls += 1
- raise asyncio.TimeoutError("transient timeout")
- return {"hits": {"hits": []}}
+ calls.append(body)
+ if len(calls) == 1:
+ raise TimeoutError("transient timeout")
+ return {"aggregations": {"filenames": {"buckets": []}}}
- _search_side_effect.md_calls = 0
opensearch_client.search.side_effect = _search_side_effect
exists = await processor.check_filename_exists("report.txt", opensearch_client)
assert exists is False
+ assert len(calls) == 2 # first attempt timed out, retry succeeded
+ assert calls[0] == calls[1] # retry sends the identical query
+ for body in calls:
+ queried = set(body["query"]["terms"]["filename"])
+ # Both the .txt name and its .md ingestion alias are covered in one query.
+ assert {"report.txt", "report.md"}.issubset(queried)
+
+
+async def test_check_filename_exists_true_when_any_alias_has_chunks():
+ processor = TaskProcessor()
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {
+ "aggregations": {"filenames": {"buckets": [{"key": "report.md", "doc_count": 3}]}}
+ }
- queried_candidates = [
- call.kwargs["body"]["query"]["term"]["filename"]
- for call in opensearch_client.search.await_args_list
- ]
- # Expected sequence:
- # 1) report.txt succeeds (no hits)
- # 2) report.md times out
- # 3) retry continues from pending alias only -> report.md
- assert queried_candidates == ["report.txt", "report.md", "report.md"]
+ assert await processor.check_filename_exists("report.txt", opensearch_client) is True
+ opensearch_client.search.assert_awaited_once()
diff --git a/tests/unit/test_shared_flag.py b/tests/unit/test_shared_flag.py
index a6c541de6..407c25894 100644
--- a/tests/unit/test_shared_flag.py
+++ b/tests/unit/test_shared_flag.py
@@ -213,12 +213,15 @@ async def test_non_cos_connector_rejects_shared_true():
@pytest.mark.asyncio
-async def test_ibm_cos_shared_true_does_not_hit_guard():
+async def test_ibm_cos_shared_true_does_not_hit_guard(monkeypatch):
"""shared=True with ibm_cos should NOT be rejected by the guard."""
from unittest.mock import AsyncMock, MagicMock
+ from api import connectors as connectors_api
from api.connectors import ConnectorSyncBody, connector_sync
+ monkeypatch.setattr(connectors_api, "has_effective_permission", AsyncMock(return_value=True))
+
body = ConnectorSyncBody(shared=True)
connector_service = MagicMock()
session_manager = MagicMock()
@@ -241,3 +244,32 @@ async def test_ibm_cos_shared_true_does_not_hit_guard():
)
# 404 = "no active connections" error, not 400 = guard rejection
assert response.status_code == 404
+
+
+async def test_ibm_cos_shared_sync_denied_without_delete_anonymous_permission(monkeypatch):
+ """shared=True requires knowledge:delete:anonymous for all sync paths."""
+ from unittest.mock import AsyncMock, MagicMock
+
+ from api import connectors as connectors_api
+ from api.connectors import ConnectorSyncBody, connector_sync
+
+ monkeypatch.setattr(connectors_api, "has_effective_permission", AsyncMock(return_value=False))
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+
+ body = ConnectorSyncBody(shared=True)
+
+ response = await connector_sync(
+ connector_type="ibm_cos",
+ body=body,
+ request=MagicMock(),
+ connector_service=MagicMock(),
+ session_manager=MagicMock(),
+ user=MagicMock(jwt_token="token", user_id="user-1"),
+ session=AsyncMock(),
+ rbac=MagicMock(),
+ )
+ import json
+
+ assert response.status_code == 403
+ assert "knowledge:delete:anonymous" in json.loads(response.body)["error"]
diff --git a/tests/unit/test_traditional_duplicate_handling.py b/tests/unit/test_traditional_duplicate_handling.py
index 30570f5e9..ea6f6ef27 100644
--- a/tests/unit/test_traditional_duplicate_handling.py
+++ b/tests/unit/test_traditional_duplicate_handling.py
@@ -1,16 +1,42 @@
-"""Unit tests for duplicate filename handling in DocumentFileProcessor."""
+"""Unit tests for the unified duplicate-filename gate across processors.
-from unittest.mock import ANY, AsyncMock, MagicMock, patch
+All processors resolve a filename duplicate the same way via
+``TaskProcessor.resolve_duplicate_filename``: with ``replace_duplicates=False``
+the file is SKIPPED (counted successful, with a warning); with
+``replace_duplicates=True`` the existing document is deleted and ingestion
+proceeds.
+"""
+
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from models.processors import DocumentFileProcessor
+from models.processors import (
+ DUPLICATE_FILENAME_WARNING,
+ DocumentFileProcessor,
+ LangflowFileProcessor,
+ S3FileProcessor,
+)
from models.tasks import FileTask, TaskStatus, UploadTask
+DUPLICATE_SKIPPED_RESULT = {
+ "status": "skipped",
+ "reason": "duplicate_filename",
+ "warning": DUPLICATE_FILENAME_WARNING,
+}
+
+
+def _assert_duplicate_skipped(upload_task: UploadTask, file_task: FileTask) -> None:
+ assert file_task.status == TaskStatus.SKIPPED
+ assert file_task.error is None
+ assert file_task.result == DUPLICATE_SKIPPED_RESULT
+ assert upload_task.failed_files == 0
+ assert upload_task.successful_files == 1
+
@pytest.mark.asyncio
async def test_traditional_processor_duplicate_exists_no_replace():
- """Verify that if a duplicate file exists and replace_duplicates is False, processing fails."""
+ """A duplicate with replace_duplicates=False is skipped, not failed."""
mock_doc_service = MagicMock()
mock_models_service = MagicMock()
mock_session_manager = MagicMock()
@@ -36,10 +62,7 @@ async def test_traditional_processor_duplicate_exists_no_replace():
await processor.process_item(upload_task, "/tmp/test.txt", file_task)
- assert file_task.status == TaskStatus.FAILED
- assert "already exists" in file_task.error
- assert upload_task.failed_files == 1
- assert upload_task.successful_files == 0
+ _assert_duplicate_skipped(upload_task, file_task)
processor.check_filename_exists.assert_called_once()
processor.delete_document_by_filename.assert_not_called()
@@ -69,7 +92,7 @@ async def test_traditional_processor_duplicate_exists_with_replace():
# Mock base class methods directly on the instance to ensure perfect isolation
processor.check_filename_exists = AsyncMock(return_value=True)
- processor.delete_document_by_filename = AsyncMock()
+ processor.delete_document_by_filename = AsyncMock(return_value=1)
processor.process_document_standard = AsyncMock(return_value={"status": "indexed"})
upload_task = UploadTask(task_id="task-123", total_files=1)
@@ -87,12 +110,179 @@ async def test_traditional_processor_duplicate_exists_with_replace():
assert upload_task.successful_files == 1
processor.check_filename_exists.assert_called_once()
- processor.delete_document_by_filename.assert_awaited_once_with(
- "test.txt",
- ANY,
- owner_user_id="user-123",
- )
+ processor.delete_document_by_filename.assert_called_once()
+ # The delete must be owner-scoped so it actually removes the old chunks.
+ delete_call = processor.delete_document_by_filename.call_args
+ assert delete_call.kwargs["owner_user_id"] == "user-123"
processor.process_document_standard.assert_called_once()
mock_session_manager.get_user_opensearch_client.assert_called_once_with(
"user-123", "mock-token"
)
+
+
+async def test_langflow_processor_duplicate_exists_no_replace():
+ """LangflowFileProcessor skips duplicates with the same uniform payload."""
+ mock_session_manager = MagicMock()
+ langflow_file_service = MagicMock()
+ langflow_file_service.upload_and_ingest_file = AsyncMock()
+
+ processor = LangflowFileProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=mock_session_manager,
+ owner_user_id="user-123",
+ jwt_token="mock-token",
+ replace_duplicates=False,
+ )
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock()
+
+ upload_task = UploadTask(task_id="task-123", total_files=1)
+ file_task = FileTask(file_path="/tmp/test.txt", filename="test.txt")
+
+ await processor.process_item(upload_task, "/tmp/test.txt", file_task)
+
+ _assert_duplicate_skipped(upload_task, file_task)
+ processor.delete_document_by_filename.assert_not_called()
+ langflow_file_service.upload_and_ingest_file.assert_not_called()
+
+
+async def test_resolve_duplicate_skips_when_delete_removes_nothing():
+ """If replace is requested but deletion removes 0 chunks (e.g. owner_user_id
+ missing on a private sync), resolve_duplicate_filename must return 'skip'
+ instead of falsely reporting 'replaced'."""
+ from models.processors import TaskProcessor
+
+ processor = TaskProcessor()
+ opensearch_client = AsyncMock()
+
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock(return_value=0)
+
+ result = await processor.resolve_duplicate_filename(
+ "report.pdf",
+ opensearch_client,
+ replace=True,
+ owner_user_id=None,
+ )
+
+ assert result == "skip"
+ processor.delete_document_by_filename.assert_awaited_once()
+
+
+async def test_delete_document_by_filename_shared_without_owner(monkeypatch):
+ """shared=True with no owner_user_id must use the anonymous filename query
+ instead of silently skipping deletion."""
+ from types import SimpleNamespace
+
+ from models.processors import TaskProcessor
+
+ admin_client = AsyncMock()
+ admin_client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(
+ "config.settings.clients",
+ SimpleNamespace(opensearch=admin_client),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
+ )
+
+ processor = TaskProcessor()
+ deleted = await processor.delete_document_by_filename(
+ "report.pdf",
+ opensearch_client,
+ owner_user_id=None,
+ shared=True,
+ )
+
+ assert deleted == 1
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ filters = search_body["query"]["bool"]["filter"]
+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters
+
+
+def _build_s3_processor(replace_duplicates: bool) -> S3FileProcessor:
+ document_service = MagicMock()
+ document_service.session_manager = MagicMock()
+ processor = S3FileProcessor(
+ document_service,
+ bucket="test-bucket",
+ s3_client=MagicMock(),
+ owner_user_id="user-123",
+ jwt_token="mock-token",
+ models_service=MagicMock(),
+ docling_service=MagicMock(),
+ replace_duplicates=replace_duplicates,
+ )
+ return processor
+
+
+async def test_s3_processor_duplicate_exists_no_replace():
+ """S3FileProcessor now has the same duplicate gate: same-key duplicates
+ are skipped before the object is even downloaded."""
+ processor = _build_s3_processor(replace_duplicates=False)
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock()
+
+ upload_task = UploadTask(task_id="task-123", total_files=1)
+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")
+
+ await processor.process_item(upload_task, "docs/report.pdf", file_task)
+
+ _assert_duplicate_skipped(upload_task, file_task)
+ processor.delete_document_by_filename.assert_not_called()
+ processor.s3_client.download_fileobj.assert_not_called()
+
+
+async def test_s3_processor_duplicate_exists_with_replace(tmp_path):
+ """With replace_duplicates=True the old document is deleted and the object
+ is downloaded and processed."""
+ processor = _build_s3_processor(replace_duplicates=True)
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock(return_value=1)
+ processor.process_document_standard = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+ processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})
+
+ upload_task = UploadTask(task_id="task-123", total_files=1)
+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")
+
+ with patch("models.processors.hash_id", return_value="dummy-hash"):
+ await processor.process_item(upload_task, "docs/report.pdf", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ assert upload_task.successful_files == 1
+ processor.delete_document_by_filename.assert_called_once()
+ assert processor.delete_document_by_filename.call_args.kwargs["owner_user_id"] == "user-123"
+ processor.s3_client.download_fileobj.assert_called_once()
+ processor.process_document_standard.assert_awaited_once()
+
+
+async def test_s3_processor_no_duplicate_proceeds():
+ """No duplicate: the gate is a no-op and processing continues."""
+ processor = _build_s3_processor(replace_duplicates=False)
+ processor.check_filename_exists = AsyncMock(return_value=False)
+ processor.delete_document_by_filename = AsyncMock()
+ processor.process_document_standard = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+ processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})
+
+ upload_task = UploadTask(task_id="task-123", total_files=1)
+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")
+
+ with patch("models.processors.hash_id", return_value="dummy-hash"):
+ await processor.process_item(upload_task, "docs/report.pdf", file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ processor.delete_document_by_filename.assert_not_called()
+ processor.process_document_standard.assert_awaited_once()