OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
refactor: Connector sync improvements (#2103)
Mike Fortman24425003e9fe
diff --git a/enhancements/connectors/azure_blob/connector.py b/enhancements/connectors/azure_blob/connector.pyindex ad0237efa..f7509553b 100644--- a/enhancements/connectors/azure_blob/connector.py+++ b/enhancements/connectors/azure_blob/connector.py@@ -8,8 +8,9 @@ so the event loop stays responsive while staying consistent with the cachedclient lifecycle used by the AWS S3 / IBM COS connectors.DLS (v1): owner-based. The connector returns an empty-principal ``DocumentACL``;-``service.process_connector_document`` assigns ``owner`` to the ingesting-OpenRAG user. Container→group principal mapping is a planned fast-follow.+``TaskProcessor.process_document_standard`` (``models/processors.py``) assigns+``owner`` to the ingesting OpenRAG user. Container→group principal mapping is+a planned fast-follow."""import asyncio@@ -68,6 +69,7 @@ class AzureBlobConnector(BaseConnector):CONNECTOR_TYPE = "azure_blob"CONNECTOR_KIND = "bucket"+ CHANGE_DETECTION = "timestamp"CONNECTOR_NAME = "Azure Blob Storage"CONNECTOR_DESCRIPTION = "Add knowledge from Azure Blob Storage"CONNECTOR_ICON = "azure-blob"@@ -334,7 +336,7 @@ class AzureBlobConnector(BaseConnector):filename = basename(blob_name) or blob_name# Owner-based DLS (v1): no per-blob principals — ownership is assigned to- # the ingesting OpenRAG user by service.process_connector_document.+ # the ingesting OpenRAG user by TaskProcessor.process_document_standard.acl = DocumentACL(owner=None, allowed_users=[], allowed_groups=[], allowed_principals=[])return ConnectorDocument(diff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.pyindex dd50eb683..6aef47cc4 100644--- a/enhancements/connectors/ibm_cos/connector.py+++ b/enhancements/connectors/ibm_cos/connector.py@@ -53,6 +53,7 @@ class IBMCOSConnector(BaseConnector):CONNECTOR_TYPE = "ibm_cos"CONNECTOR_KIND = "bucket"+ CHANGE_DETECTION = "timestamp"CONNECTOR_NAME = "IBM Cloud Object Storage"CONNECTOR_DESCRIPTION = "Add knowledge from IBM Cloud Object Storage"CONNECTOR_ICON = "ibm-cos"diff --git a/src/api/connectors.py b/src/api/connectors.pyindex ba83bf843..6f25e7573 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -22,7 +22,6 @@ from services.connector_access_service import (CONNECTOR_TYPES,filter_connectors_for_user,get_access_map,- is_bucket_connector_type,is_connector_access_policy_enforced,is_connector_allowed_for_request,list_access_for_admin,@@ -73,8 +72,33 @@ async def _allowed_connector_types_for_request(def _connector_sync_should_replace(connector_type: str) -> bool:- """Return True for connector types where sync should replace existing indexed files."""- return connector_type in ["google_drive", "sharepoint", "onedrive"]+ """Return True when sync should replace existing indexed files for this+ connector type, so content changes propagate on re-sync.++ Declared per connector via ``BaseConnector.SYNC_REPLACES_DUPLICATES``+ (default True) instead of a hardcoded type list, so new connectors get+ change-propagating sync without touching this module. Unknown types (e.g.+ stale index docs from a removed enhancement connector) stay conservative+ and skip duplicates.+ """+ from connectors.registry import get_connector_class++ cls = get_connector_class(connector_type)+ return cls.SYNC_REPLACES_DUPLICATES if cls is not None else False+++def _connector_uses_timestamp_change_detection(connector_type: str) -> bool:+ """True when sync should re-ingest only files whose remote modified_time is+ newer than the stored one (see bucket_changed_file_ids), instead of+ replacing every indexed file.++ Declared per connector via ``BaseConnector.CHANGE_DETECTION`` — the bucket+ connectors set ``"timestamp"``; the default is ``"replace_always"``.+ """+ from connectors.registry import get_connector_class++ cls = get_connector_class(connector_type)+ return cls is not None and cls.CHANGE_DETECTION == "timestamp"def _is_unmapped_keyword_agg_error(err: Exception) -> bool:@@ -104,8 +128,8 @@ async def get_synced_file_ids_for_connector(source ID.- ``filenames``: unique filenames as a fallback when ``file_ids`` is empty.- ``id_field``: the OpenSearch field name that ``file_ids`` came from- (``"connector_file_id"`` or ``"document_id"``). Callers must pass this to- ``delete_orphan_documents`` so deletions target the correct field.+ (``"connector_file_id"`` or ``"document_id"``). Informational (logging);+ chunk deletion matches both fields via ``connectors.chunk_cleanup``."""try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)@@ -473,28 +497,29 @@ async def compute_orphans_for_connector_type()return None- # Drive the per-id existence check via cfg.file_ids when the- # connector supports it (SharePoint / OneDrive / Google Drive).+ # Drive the per-id existence check via list_selected_files when+ # the connector supports it (SharePoint / OneDrive / Google Drive).# The flat default of list_files() only returns the *root* listing# (e.g. /drive/root/children for SharePoint, files-only, no folder# traversal), so any folder-internal file in OpenSearch would be# absent from remote_ids and wrongly flagged as an orphan.- # _list_selected_files iterates each id via _get_file_metadata_by_id+ # list_selected_files iterates each id via _get_file_metadata_by_id# and silently drops missing ids, so the resulting `remote_ids` is# exactly "the subset of existing_file_ids that still exists at# source" — which is what orphan detection actually needs.- cfg = getattr(connector, "cfg", None)- scoped_listing = cfg is not None and bool(existing_file_ids)+ # cfg is None on bucket connectors (BaseConnector declares it as a+ # class default), so guard on cfg-is-not-None rather than hasattr:+ # otherwise bucket connectors route through list_selected_files ->+ # list_files() (the whole account) instead of the flat listing below.+ scoped_listing = getattr(connector, "cfg", None) is not None and bool(existing_file_ids)- original_file_ids = None- original_folder_ids = Noneif scoped_listing:- original_file_ids = getattr(cfg, "file_ids", None)- original_folder_ids = getattr(cfg, "folder_ids", None)- cfg.file_ids = list(existing_file_ids)- cfg.folder_ids = None-- try:+ page = await connector.list_selected_files(list(existing_file_ids))+ for f in page.get("files", []):+ fid = f.get("id")+ if fid:+ remote_ids.add(fid)+ else:page_token = Nonewhile True:page = await connector.list_files(page_token=page_token)@@ -505,10 +530,6 @@ async def compute_orphans_for_connector_type(page_token = page.get("nextPageToken") or page.get("next_page_token")if not page_token:break- finally:- if scoped_listing:- cfg.file_ids = original_file_ids- cfg.folder_ids = original_folder_idsexcept Exception as e:logger.warning("Skipping orphan compute — listing failed",@@ -531,31 +552,36 @@ async def delete_orphan_documents(user_id: str,session_manager,jwt_token: str | None,- id_field: str = "document_id",+ *,+ connector_type: str | None = None,+ shared: bool = False,) -> int:"""Delete OpenSearch chunks for the given orphan IDs. Returns the number ofchunks deleted (0 on failure).- ``id_field`` must match the OpenSearch field that ``orphan_ids`` came from —- either ``"connector_file_id"`` (ConnectorFileProcessor / non-Langflow path)- or ``"document_id"`` (Langflow path, where document_id holds the connector- source ID). Pass the value returned as the third element of- ``get_synced_file_ids_for_connector()``.+ Deletion matches both id layouts (``connector_file_id`` for the standard+ ingest path, ``document_id`` for the Langflow path) via+ ``connectors.chunk_cleanup``, so callers no longer need to track which+ field the ids came from."""if not orphan_ids:return 0- from .documents import delete_chunks_by_document_ids+ from connectors.chunk_cleanup import delete_connector_file_chunkstry:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- return await delete_chunks_by_document_ids(- orphan_ids, opensearch_client, get_index_name(), field=id_field+ return await delete_connector_file_chunks(+ orphan_ids,+ opensearch_client,+ connector_type=connector_type,+ owner_user_id=None if shared else user_id,+ shared=shared,+ refresh=True,)except Exception as e:logger.error("Orphan delete failed",orphan_count=len(orphan_ids),- id_field=id_field,error=str(e),)return 0@@ -568,16 +594,13 @@ async def reconcile_orphans_for_connector_type(session_manager,jwt_token: str | None,existing_file_ids: list[str],- id_field: str = "document_id",+ *,+ shared: bool = False,) -> list[str]:"""Compute and delete orphans for a connector type. Thin wrapper aroundcompute_orphans_for_connector_type + delete_orphan_documents preserved forcallers that perform sync immediately after reconcile.- ``id_field`` must match the OpenSearch field that ``existing_file_ids`` came- from. Pass the value returned as the third element of- ``get_synced_file_ids_for_connector()``.-Returns the list of orphan file IDs that were deleted (or [])."""orphans = await compute_orphans_for_connector_type(@@ -597,20 +620,138 @@ async def reconcile_orphans_for_connector_type(user_id=user_id,session_manager=session_manager,jwt_token=jwt_token,- id_field=id_field,+ connector_type=connector_type,+ shared=shared,)logger.info("Orphan reconcile complete",connector_type=connector_type,orphan_count=len(orphan_ids),deleted_chunks=deleted,- id_field=id_field,)if deleted <= 0:return []return orphan_ids+async def _sync_existing_connector_files(+ connector_type: str,+ working_connection,+ user_id: str,+ connector_service,+ session_manager,+ jwt_token: str | None,+ existing_file_ids: list[str],+ existing_filenames: list[str],+ id_field: str,+ *,+ ingest_settings: dict[str, Any] | None = None,+ shared: bool = False,+ reconcile: bool = True,+ max_files: int | None = None,+) -> dict[str, Any]:+ """Re-sync the files already indexed for a connector type — the shared+ no-selection Sync flow used by both ``connector_sync`` and+ ``sync_all_connectors``.++ Orphans (deleted at the source) are reconciled first when ``reconcile`` is+ True, then either timestamp change detection (updates-only re-ingest) or a+ full re-sync runs depending on the connector's ``CHANGE_DETECTION``+ capability; connectors with no stored ids fall back to filename filtering.++ Returns an outcome dict the caller maps onto its own response shape:+ * ``{"outcome": "synced", "task_id": ...}``+ * ``{"outcome": "deleted_only"}`` — orphan cleanup removed every file;+ nothing left to sync.+ * ``{"outcome": "up_to_date"}`` — timestamp change detection found no+ remote changes.+ """+ if existing_file_ids:+ logger.info(+ "Syncing specific files by connector file ID",+ connector_type=connector_type,+ file_count=len(existing_file_ids),+ id_field=id_field,+ )+ # Reconcile orphans (files deleted at the source) before re-syncing.+ # Callers gate this: a capped sync sees a partial remote listing and+ # would delete legitimate files.+ ids_to_sync = list(existing_file_ids)+ if reconcile:+ orphan_ids = await reconcile_orphans_for_connector_type(+ connector_type=connector_type,+ user_id=user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ shared=shared,+ )+ if orphan_ids:+ orphan_id_set = set(orphan_ids)+ ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]+ if not ids_to_sync:+ return {"outcome": "deleted_only"}+ if _connector_uses_timestamp_change_detection(connector_type):+ # Timestamp Sync is updates-only: re-ingest just the files whose+ # remote copy is newer than what's indexed (deleting the stale+ # chunks via replace_duplicates). Unlike replace_duplicates=False+ # this actually propagates content changes; unlike replacing every+ # id it skips unchanged files instead of re-fetching the source.+ connector = await connector_service.get_connector(working_connection.connection_id)+ changed_ids = await bucket_changed_file_ids(+ connector,+ connector_type,+ user_id,+ session_manager,+ jwt_token,+ ids_to_sync,+ )+ if not changed_ids:+ return {"outcome": "up_to_date"}+ if max_files is not None:+ changed_ids = changed_ids[:max_files]+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user_id,+ changed_ids,+ jwt_token=jwt_token,+ ingest_settings=ingest_settings,+ replace_duplicates=True,+ shared=shared,+ )+ else:+ if max_files is not None:+ ids_to_sync = ids_to_sync[:max_files]+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user_id,+ ids_to_sync,+ jwt_token=jwt_token,+ ingest_settings=ingest_settings,+ replace_duplicates=_connector_sync_should_replace(connector_type),+ shared=shared,+ )+ else:+ # Fallback: use filename filtering (for Langflow-ingested files without document_id)+ logger.info(+ "Syncing files by filename filter (document_id not available)",+ connector_type=connector_type,+ filename_count=len(existing_filenames),+ )+ task_id = await connector_service.sync_connector_files(+ working_connection.connection_id,+ user_id,+ max_files=max_files,+ jwt_token=jwt_token,+ filename_filter=set(existing_filenames),+ ingest_settings=ingest_settings,+ replace_duplicates=_connector_sync_should_replace(connector_type),+ shared=shared,+ )+ return {"outcome": "synced", "task_id": task_id}++class ConnectorSyncBody(BaseModel):max_files: int | None = Noneselected_files: list[Any] | None = None@@ -675,16 +816,12 @@ async def _expand_selected_connector_files(file_ids = [f.get("id") for f in selected_files_raw if isinstance(f, dict) and f.get("id")]expanded_files_info: list[dict[str, Any]] = []- if file_ids and hasattr(connector, "cfg"):- # The connector is cached and shared across requests by- # ConnectionManager, so scope the listing to a per-call copy with its- # own cfg instead of mutating (and racing on) the shared config.- scoped_connector = copy.copy(connector)- scoped_connector.cfg = copy.copy(connector.cfg)- scoped_connector.cfg.file_ids = file_ids- scoped_connector.cfg.folder_ids = None+ # cfg is None on bucket connectors (BaseConnector class default), so guard on+ # cfg-is-not-None: only cfg-backed connectors expand folders here; bucket+ # connectors fall through to using selected_files_raw directly below.+ if file_ids and getattr(connector, "cfg", None) is not None:try:- result = await scoped_connector.list_files()+ result = await connector.list_selected_files(file_ids)for f in result.get("files", []):expanded_files_info.append(_connector_file_response(f))except Exception as e:@@ -736,21 +873,14 @@ async def _classify_connector_duplicates("total_files": len(cleaned_files),}- opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- query_body = {- "size": 10000,- "query": {"terms": {"filename": list(all_candidates)}},- "_source": ["filename"],- }+ from utils.opensearch_filenames import find_existing_filenames+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)existing_filenames = set()try:- response = await opensearch_client.search(index=get_index_name(), body=query_body)- hits = response.get("hits", {}).get("hits", [])- for hit in hits:- fn = hit.get("_source", {}).get("filename")- if fn:- existing_filenames.add(fn)+ existing_filenames = await find_existing_filenames(+ all_candidates, opensearch_client, get_index_name()+ )except Exception as search_err:if "index_not_found_exception" not in str(search_err):raise@@ -1158,6 +1288,14 @@ async def connector_sync(status_code=400,)+ if body.shared and not await has_effective_permission(+ request, user, rbac, "knowledge:delete:anonymous"+ ):+ return JSONResponse(+ {"error": "Shared sync requires the knowledge:delete:anonymous permission"},+ status_code=403,+ )+# Get all active connections for this connector type and userconnections = await connector_service.connection_manager.list_connections(user_id=user.user_id, connector_type=connector_type@@ -1220,17 +1358,6 @@ async def connector_sync(# Explicit files selected (e.g., from file picker) - sync those specific filesfrom .documents import _ensure_index_exists- if body.shared and body.replace_duplicates:- if not await has_effective_permission(- request, user, rbac, "knowledge:delete:anonymous"- ):- return JSONResponse(- {- "error": "Replacing shared documents requires the knowledge:delete:anonymous permission"- },- status_code=403,- )-if not body.replace_duplicates and file_infos:duplicate_check = await _classify_connector_duplicates(connector=await connector_service.get_connector(@@ -1419,101 +1546,40 @@ async def connector_sync(status_code=200,)- # If we have connector file IDs, use sync_specific_files- # Otherwise, use filename filtering with sync_connector_files- if existing_file_ids:- logger.info(- "Syncing specific files by connector file ID",- connector_type=connector_type,- file_count=len(existing_file_ids),- id_field=id_field,- )- # Reconcile orphans (files deleted at the source) before re-syncing.- # Strict gating: skip when sync is capped — we'd see a partial remote- # listing and delete legitimate files.- ids_to_sync = list(existing_file_ids)- if body.max_files is None:- orphan_ids = await reconcile_orphans_for_connector_type(- connector_type=connector_type,- user_id=user.user_id,- connector_service=connector_service,- session_manager=session_manager,- jwt_token=jwt_token,- existing_file_ids=existing_file_ids,- id_field=id_field,- )- if orphan_ids:- orphan_id_set = set(orphan_ids)- ids_to_sync = [fid for fid in existing_file_ids if fid not in orphan_id_set]- if not ids_to_sync:- return JSONResponse(- {- "status": "no_files",- "message": f"Deleted stale {connector_type} files; no remaining files to sync.",- },- status_code=200,- )- if is_bucket_connector_type(connector_type):- # Bucket Sync is updates-only: re-ingest just the blobs whose- # remote copy is newer than what's indexed (deleting the stale- # chunks via replace_duplicates). Unlike replace_duplicates=False- # this actually propagates content changes; unlike replacing every- # id it skips unchanged blobs instead of re-fetching the container.- connector = await connector_service.get_connector(- working_connection.connection_id- )- changed_ids = await bucket_changed_file_ids(- connector,- connector_type,- user.user_id,- session_manager,- jwt_token,- ids_to_sync,- )- if not changed_ids:- return JSONResponse(- {- "status": "no_files",- "message": f"All {connector_type} files are already up to date.",- },- status_code=200,- )- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- changed_ids,- jwt_token=jwt_token,- ingest_settings=body.settings,- replace_duplicates=True,- shared=body.shared,- )- else:- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- ids_to_sync,- jwt_token=jwt_token,- ingest_settings=body.settings,- replace_duplicates=_connector_sync_should_replace(connector_type),- shared=body.shared,- )- else:- # Fallback: use filename filtering (for Langflow-ingested files without document_id)- logger.info(- "Syncing files by filename filter (document_id not available)",- connector_type=connector_type,- filename_count=len(existing_filenames),+ sync_result = await _sync_existing_connector_files(+ connector_type=connector_type,+ working_connection=working_connection,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ existing_filenames=existing_filenames,+ id_field=id_field,+ ingest_settings=body.settings,+ shared=body.shared,+ # Strict gating: skip orphan reconcile when sync is capped — we'd+ # see a partial remote listing and delete legitimate files.+ reconcile=body.max_files is None,+ max_files=body.max_files,+ )+ if sync_result["outcome"] == "deleted_only":+ return JSONResponse(+ {+ "status": "no_files",+ "message": f"Deleted stale {connector_type} files; no remaining files to sync.",+ },+ status_code=200,)- task_id = await connector_service.sync_connector_files(- working_connection.connection_id,- user.user_id,- max_files=None,- jwt_token=jwt_token,- filename_filter=set(existing_filenames),- ingest_settings=body.settings,- replace_duplicates=_connector_sync_should_replace(connector_type),- shared=body.shared,+ if sync_result["outcome"] == "up_to_date":+ return JSONResponse(+ {+ "status": "no_files",+ "message": f"All {connector_type} files are already up to date.",+ },+ status_code=200,)+ task_id = sync_result["task_id"]# The bucket_filter path may have already populated task_ids (new + changed# batches); every other branch sets a single task_id.if not task_ids:@@ -2020,82 +2086,25 @@ async def sync_all_connectors()continue- # Sync using connector file IDs if available, else use filename filter- if existing_file_ids:- logger.info(- "Syncing specific files by connector file ID",- connector_type=connector_type,- file_count=len(existing_file_ids),- id_field=id_field,- )- # Reconcile orphans (files deleted at the source) before re-syncing.- # sync_all_connectors has no caps or filters, so gating reduces- # to the strict checks inside the helper.- orphan_ids = await reconcile_orphans_for_connector_type(- connector_type=connector_type,- user_id=user.user_id,- connector_service=connector_service,- session_manager=session_manager,- jwt_token=jwt_token,- existing_file_ids=existing_file_ids,- id_field=id_field,- )- if orphan_ids:- orphan_id_set = set(orphan_ids)- existing_file_ids = [- fid for fid in existing_file_ids if fid not in orphan_id_set- ]- if not existing_file_ids:- deleted_only_connectors.append(connector_type)- continue- if is_bucket_connector_type(connector_type):- # Updates-only change detection (see connector_sync): re-ingest- # only blobs that changed at source, replacing stale chunks.- connector = await connector_service.get_connector(- working_connection.connection_id- )- changed_ids = await bucket_changed_file_ids(- connector,- connector_type,- user.user_id,- session_manager,- jwt_token,- existing_file_ids,- )- if not changed_ids:- # Nothing changed at source — already up to date.- skipped_connectors.append(connector_type)- continue- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- changed_ids,- jwt_token=jwt_token,- replace_duplicates=True,- )- else:- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- existing_file_ids,- jwt_token=jwt_token,- replace_duplicates=_connector_sync_should_replace(connector_type),- )- else:- # Fallback: use filename filtering- logger.info(- "Syncing files by filename filter",- connector_type=connector_type,- filename_count=len(existing_filenames),- )- task_id = await connector_service.sync_connector_files(- working_connection.connection_id,- user.user_id,- max_files=None,- jwt_token=jwt_token,- filename_filter=set(existing_filenames),- replace_duplicates=_connector_sync_should_replace(connector_type),- )+ sync_result = await _sync_existing_connector_files(+ connector_type=connector_type,+ working_connection=working_connection,+ user_id=user.user_id,+ connector_service=connector_service,+ session_manager=session_manager,+ jwt_token=jwt_token,+ existing_file_ids=existing_file_ids,+ existing_filenames=existing_filenames,+ id_field=id_field,+ )+ if sync_result["outcome"] == "deleted_only":+ deleted_only_connectors.append(connector_type)+ continue+ if sync_result["outcome"] == "up_to_date":+ # Nothing changed at source — already up to date.+ skipped_connectors.append(connector_type)+ continue+ task_id = sync_result["task_id"]all_task_ids.append(task_id)synced_connectors.append(connector_type)diff --git a/src/api/documents.py b/src/api/documents.pyindex c614dd7c1..f4901dace 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -198,45 +198,6 @@ async def delete_documents_by_filename_core()-async def delete_chunks_by_document_ids(- document_ids: list[str],- opensearch_client,- index_name: str,- write_opensearch_client=None,- field: str = "document_id",-) -> int:- """Bulk delete OpenSearch chunks by a keyword field. Returns deleted count.-- DLS-safe: enumerate the visible chunk _ids via search, then issue a trusted- delete per primary id. `delete_by_query` is silently no-opped under DLS- (returns deleted:N but leaves docs in place).-- `field` selects which indexed keyword to match against (default: ``document_id``).- Pass ``field="connector_file_id"`` to clean up chunks for a deleted connector- source file when the connector file ID differs from the content hash stored in- ``document_id``.- """- if not document_ids:- return 0- from config.settings import clients- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids-- chunk_ids = await collect_visible_document_ids(- opensearch_client,- index=index_name,- query={"terms": {field: document_ids}},- )- write_client = write_opensearch_client or clients.opensearch- if write_client is None:- raise RuntimeError("Backend OpenSearch write client is unavailable")- return await delete_document_ids(- write_client,- index=index_name,- document_ids=chunk_ids,- refresh=True,- )--async def _ensure_index_exists(jwt_token: str = None):"""Create the OpenSearch index if it doesn't exist yet."""from config.settings import clients as app_clients@@ -263,24 +224,13 @@ async def check_filename_exists(try:opensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)- from utils.file_utils import get_filename_aliases- from utils.opensearch_queries import build_filename_search_body-- candidate_filenames = get_filename_aliases(filename)- if not candidate_filenames:- return JSONResponse({"exists": False, "filename": filename}, status_code=200)+ from utils.opensearch_filenames import filename_existslogger.debug("Checking filename existence", filename=filename, index_name=get_index_name())exists = Falsetry:- for candidate in candidate_filenames:- search_body = build_filename_search_body(candidate, size=1, source=["filename"])- response = await opensearch_client.search(index=get_index_name(), body=search_body)- hits = response.get("hits", {}).get("hits", [])- if hits:- exists = True- break+ exists = await filename_exists(filename, opensearch_client, get_index_name())except Exception as search_err:if "index_not_found_exception" in str(search_err):logger.info("Index does not exist, creating it now before upload")diff --git a/src/api/upload.py b/src/api/upload.pyindex e2712c8a0..cb2590e4a 100644--- a/src/api/upload.py+++ b/src/api/upload.py@@ -30,6 +30,9 @@ class UploadPathBody(BaseModel):class UploadBucketBody(BaseModel):s3_url: str+ # When True, objects whose filename (S3 key) already exists in the index+ # replace the indexed copy; when False (default) they are skipped.+ replace_duplicates: bool = Falseasync def upload(@@ -230,6 +233,7 @@ async def upload_bucket(jwt_token=jwt_token,owner_name=owner_name,owner_email=owner_email,+ replace_duplicates=body.replace_duplicates,)task_id = await task_service.create_custom_task(task_user_id, keys, processor)diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.pyindex 12377d2b5..8267a3879 100644--- a/src/connectors/aws_s3/connector.py+++ b/src/connectors/aws_s3/connector.py@@ -47,6 +47,7 @@ class S3Connector(BaseConnector):CONNECTOR_TYPE = "aws_s3"CONNECTOR_KIND = "bucket"+ CHANGE_DETECTION = "timestamp"CONNECTOR_NAME = "Amazon S3"CONNECTOR_DESCRIPTION = "Add knowledge from Amazon S3 or any S3-compatible storage"CONNECTOR_ICON = "aws-s3"diff --git a/src/connectors/base.py b/src/connectors/base.pyindex 48b77b312..9b7a420f9 100644--- a/src/connectors/base.py+++ b/src/connectors/base.py@@ -1,3 +1,4 @@+import copyimport osfrom abc import ABC, abstractmethodfrom dataclasses import dataclass@@ -64,6 +65,23 @@ class BaseConnector(ABC):# "oauth" connectors authenticate per-user via OAuth env-var credentials.# "bucket" connectors authenticate via per-connection config dict (HMAC, API key, etc).CONNECTOR_KIND: str = "oauth"+ # When a sync re-processes files that are already indexed, pass+ # replace_duplicates so the indexed copy is replaced and content changes+ # propagate, instead of being skipped by the duplicate-filename gate.+ # Default True so a new connector gets change-propagating sync out of the+ # box (the content-hash "unchanged" short-circuit still avoids re-indexing+ # identical bytes on the non-Langflow path). Bucket connectors normally+ # never consult this: their sync paths divert to modified_time change+ # detection and replace only the blobs that actually changed.+ SYNC_REPLACES_DUPLICATES: bool = True+ # How sync decides whether an already-indexed file needs re-ingesting:+ # "replace_always" — re-process every indexed file on sync, replacing the+ # indexed copy (the content-hash short-circuit still+ # skips identical bytes on the non-Langflow path).+ # "timestamp" — list the source once and re-ingest only files whose+ # remote modified_time is strictly newer than the+ # stored one (see bucket_changed_file_ids).+ CHANGE_DETECTION: str = "replace_always"# Connector-specific keys in the config dict that must be encrypted at rest.SECRET_CONFIG_KEYS: tuple = ()@@ -72,6 +90,11 @@ class BaseConnector(ABC):CONNECTOR_DESCRIPTION: str = NoneCONNECTOR_ICON: str = None # Icon identifier or emoji+ # Per-connection settings (file_ids/folder_ids selection, etc.). Declared+ # here so subclasses that set a concrete cfg type still satisfy the base+ # type; list_selected_files() scopes it via a per-call shallow copy.+ cfg: Any = None+def __init__(self, config: dict[str, Any]):self.config = configself._authenticated = False@@ -175,6 +198,40 @@ class BaseConnector(ABC):"""List all files. Returns files and next_page_token if any."""pass+ async def list_selected_files(+ self,+ file_ids: list[str],+ folder_ids: list[str] | None = None,+ **kwargs: Any,+ ) -> dict[str, Any]:+ """List specific files/folders by scoping cfg to this call.++ Callers that need list_files() to operate on a specific set of ids+ previously had to manually save cfg.file_ids / cfg.folder_ids, overwrite+ them, call list_files(), then restore in a finally block. This method+ owns that scoping so the caller never has to.++ Connectors are cached and shared across requests by ConnectionManager,+ so we operate on a per-call shallow copy with its own cfg instead of+ mutating (and racing on) the shared config.+ """+ cfg = getattr(self, "cfg", None)+ if cfg is None:+ # No per-call selection state to scope by. Falling back to+ # list_files() would list the connector's entire account and+ # silently ignore file_ids, so refuse instead: callers must gate on+ # `cfg is not None` and use file_ids directly for cfg-less+ # (bucket) connectors.+ raise NotImplementedError(+ f"{type(self).__name__} has no cfg; list_selected_files is only "+ "supported on cfg-backed connectors (Google Drive/OneDrive/SharePoint)."+ )+ scoped = copy.copy(self)+ scoped.cfg = copy.copy(cfg)+ scoped.cfg.file_ids = file_ids+ scoped.cfg.folder_ids = folder_ids+ return await scoped.list_files(**kwargs)+@abstractmethodasync def get_file_content(self, file_id: str) -> ConnectorDocument:"""Get file content and metadata"""diff --git a/src/connectors/chunk_cleanup.py b/src/connectors/chunk_cleanup.pynew file mode 100644index 000000000..6ef2d3aa0--- /dev/null+++ b/src/connectors/chunk_cleanup.py@@ -0,0 +1,110 @@+"""Single deletion path for a connector file's indexed chunks.++Deletion propagation has multiple *triggers* — orphan reconciliation (file+missing from a full remote listing), per-file 404 cleanup during sync, rename+cleanup — but exactly one deletion *semantic*, owned here:++* Chunks are matched by the stable connector file id against BOTH id layouts:+ the standard ingest path stores it in ``connector_file_id`` (``document_id``+ holds the content hash), the Langflow path stores it in ``document_id``.+ Matching a single field misses chunks from the other layout.+* Optionally owner-scoped: ``owner_user_id`` restricts to that owner's chunks;+ ``shared=True`` targets ownerless (instance-shared) chunks instead.+* DLS-safe: visible chunk ``_id``s are enumerated with the caller's+ (user-scoped) client, then deleted individually with the trusted backend+ write client — ``delete_by_query`` is silently no-opped under DLS.+"""++from collections.abc import Iterable+from typing import Any+++def build_connector_file_chunks_query(+ file_ids: Iterable[str],+ *,+ connector_type: str | None = None,+ owner_user_id: str | None = None,+ shared: bool = False,+ keep_filenames: Iterable[str] | None = None,+) -> dict[str, Any]:+ """Query matching every chunk of the given connector file ids.++ ``connector_type`` narrows the match to a single connector type so IDs+ that happen to collide across connectors are not affected.++ ``keep_filenames`` excludes chunks indexed under those names — used by+ rename cleanup to drop only the stale old-name chunks of a file id.+ """+ ids = [fid for fid in file_ids if fid]+ filters: list[dict[str, Any]] = [+ {+ "bool": {+ "should": [+ {"terms": {"document_id": ids}},+ {"terms": {"connector_file_id": ids}},+ # Some indices predate the explicit keyword mapping for+ # connector_file_id, so it was dynamically mapped as+ # analyzed text with a .keyword multi-field.+ {"terms": {"connector_file_id.keyword": ids}},+ ],+ "minimum_should_match": 1,+ }+ }+ ]+ if connector_type:+ filters.append({"term": {"connector_type": connector_type}})+ if shared:+ filters.append({"bool": {"must_not": {"exists": {"field": "owner"}}}})+ elif owner_user_id:+ filters.append({"term": {"owner": owner_user_id}})+ query: dict[str, Any] = {"bool": {"filter": filters}}+ keep = [name for name in (keep_filenames or []) if name]+ if keep:+ query["bool"]["must_not"] = [{"terms": {"filename": keep}}]+ return query+++async def delete_connector_file_chunks(+ file_ids: Iterable[str],+ opensearch_client,+ *,+ connector_type: str | None = None,+ owner_user_id: str | None = None,+ shared: bool = False,+ keep_filenames: Iterable[str] | None = None,+ refresh: bool = False,+) -> int:+ """Delete the indexed chunks of the given connector file ids.++ Returns the number of chunks deleted. Raises on infrastructure failure+ (missing write client, OpenSearch errors) — callers that must never fail+ their surrounding task wrap this in their own best-effort handling.+ """+ from config.settings import clients, get_index_name+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids++ ids = [fid for fid in (file_ids or []) if fid]+ if not ids:+ return 0++ write_client = clients.opensearch+ if write_client is None:+ raise RuntimeError("Backend OpenSearch write client is unavailable")++ chunk_ids = await collect_visible_document_ids(+ opensearch_client,+ index=get_index_name(),+ query=build_connector_file_chunks_query(+ ids,+ connector_type=connector_type,+ owner_user_id=owner_user_id,+ shared=shared,+ keep_filenames=keep_filenames,+ ),+ )+ return await delete_document_ids(+ write_client,+ index=get_index_name(),+ document_ids=chunk_ids,+ refresh=refresh,+ )diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 421cb8015..c91f4c95d 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -1,6 +1,6 @@from typing import Any-from utils.file_utils import clean_connector_filename, get_file_extension+from utils.file_utils import clean_connector_filenamefrom utils.logging_config import get_loggerfrom .base import BaseConnector, ConnectorDocument@@ -76,171 +76,6 @@ class ConnectorService:return self.session_manager.create_jwt_token(user)return self.session_manager.get_effective_jwt_token(user.user_id, effective_token)- async def process_connector_document(- self,- document: ConnectorDocument,- owner_user_id: str,- connector_type: str,- jwt_token: str = None,- owner_name: str = None,- owner_email: str = None,- ingest_settings: dict[str, Any] | None = None,- ) -> dict[str, Any]:- """Process a document from a connector using active processing pipeline"""- jwt_token = await self._get_effective_sync_jwt(owner_user_id, jwt_token)-- from config.settings import DISABLE_INGEST_WITH_LANGFLOW-- if not DISABLE_INGEST_WITH_LANGFLOW and self.langflow_service is not None:- # Process via Langflow pipeline- from utils.file_utils import langflow_safe_filename_and_mimetype-- langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(- document.filename, document.mimetype- )- file_tuple = (langflow_filename, document.content, processed_mimetype)-- allowed_users = []- allowed_groups = []- allowed_principals = []- allowed_principal_labels = []- if document.acl:- try:- allowed_users = document.acl.allowed_users or []- allowed_groups = document.acl.allowed_groups or []- allowed_principals = document.acl.allowed_principals or []- allowed_principal_labels = document.acl.allowed_principal_labels or []- except AttributeError:- pass-- connector_tweak_settings = None- if isinstance(ingest_settings, dict):- connector_tweak_settings = dict(ingest_settings)- connector_tweak_settings.pop("embeddingModel", None)-- tweaks = self.langflow_service.merge_ui_ingest_settings_into_tweaks(- {}, connector_tweak_settings- )-- result = await self.langflow_service.upload_and_ingest_file(- file_tuple=file_tuple,- session_id=None,- tweaks=tweaks,- settings=ingest_settings,- jwt_token=jwt_token,- owner=owner_user_id,- owner_name=owner_name,- owner_email=owner_email,- connector_type=connector_type,- docling_polling_service=self.task_service.docling_polling_service- if self.task_service- else None,- connector_file_id=document.id,- source_url=document.source_url,- allowed_users=allowed_users,- allowed_groups=allowed_groups,- allowed_principals=allowed_principals,- allowed_principal_labels=allowed_principal_labels,- original_filename=document.filename,- original_mimetype=document.mimetype,- )- return {- "status": "indexed",- "filename": document.filename,- "source_url": document.source_url,- "document_id": document.id,- "connector_type": connector_type,- "langflow_result": result,- }- else:- # Create temporary file from document content- import os-- from utils.file_utils import auto_cleanup_tempfile-- suffix = os.path.splitext(document.filename)[1]- if not suffix:- suffix = get_file_extension(document.mimetype)-- with auto_cleanup_tempfile(suffix=suffix) as tmp_path:- # Write document content to temp file- with open(tmp_path, "wb") as f:- f.write(document.content)-- logger.info(- "[CONNECTOR] Processing document",- document_id=document.id,- connector_type=connector_type,- filename=document.filename,- )-- # Process using consolidated processing pipeline- from models.processors import TaskProcessor-- processor = TaskProcessor(- document_service=self.document_service,- models_service=self.models_service,- docling_service=self.docling_service,- )- standard_kwargs: dict[str, Any] = {}- if isinstance(ingest_settings, dict):- em = ingest_settings.get("embeddingModel")- if isinstance(em, str) and em.strip():- standard_kwargs["embedding_model"] = em.strip()- for ui_key, param in (- ("chunkSize", "chunk_size"),- ("chunkOverlap", "chunk_overlap"),- ):- raw = ingest_settings.get(ui_key)- if raw is not None:- try:- standard_kwargs[param] = int(raw)- except (TypeError, ValueError):- pass- if "ocr" in ingest_settings:- standard_kwargs["ocr"] = bool(ingest_settings["ocr"])- if "pictureDescriptions" in ingest_settings:- standard_kwargs["picture_descriptions"] = bool(- ingest_settings["pictureDescriptions"]- )-- result = await processor.process_document_standard(- file_path=tmp_path,- file_hash=document.id,- owner_user_id=owner_user_id,- original_filename=document.filename,- jwt_token=jwt_token,- owner_name=owner_name,- owner_email=owner_email,- file_size=len(document.content) if document.content else 0,- connector_type=connector_type,- acl=document.acl,- connector_file_id=document.id,- **standard_kwargs,- )-- logger.info(- "[CONNECTOR] Document processed",- document_id=document.id,- status=result.get("status"),- )-- # If successfully indexed or already exists, update the indexed documents with connector metadata- if result["status"] in ["indexed", "unchanged"]:- # Update all chunks with connector-specific metadata- await self._update_connector_metadata(- document,- owner_user_id,- connector_type,- jwt_token,- )-- return {- **result,- "filename": document.filename,- "source_url": document.source_url,- }-async def _update_connector_metadata(self,document: ConnectorDocument,@@ -546,31 +381,23 @@ class ConnectorService:connector.set_file_infos(file_infos)logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")- # Temporarily set file_ids in the connector's config so list_files() can use them- # Store the original values to restore later- original_file_ids = None- original_folder_ids = None-- if hasattr(connector, "cfg"):- original_file_ids = getattr(connector.cfg, "file_ids", None)- original_folder_ids = getattr(connector.cfg, "folder_ids", None)-expanded_file_ids = file_ids # Default to original IDsexpanded_files_info = []try:- # Set the file_ids we want to sync in the connector's config- if hasattr(connector, "cfg"):- connector.cfg.file_ids = file_ids- connector.cfg.folder_ids = None-- # Get the expanded list of file IDs (folders will be expanded to their contents)- # This uses the connector's list_files() which calls _iter_selected_items()- result = await connector.list_files()+ # cfg is None on bucket connectors (azure_blob/aws_s3/ibm_cos): they+ # have no per-call file/folder selection to expand, and file_ids are+ # already the exact ids to sync. Only cfg-backed connectors+ # (Google Drive/OneDrive/SharePoint) expand folders here. Guarding on+ # cfg-is-not-None rather than hasattr is deliberate: BaseConnector+ # declares cfg=None as a class default, so hasattr is True for every+ # connector and would route bucket syncs through list_selected_files+ # -> list_files() (the whole account), discarding the selected ids.+ if getattr(connector, "cfg", None) is not None:+ result = await connector.list_selected_files(file_ids)expanded_files = result.get("files", [])expanded_file_ids = [f["id"] for f in expanded_files]- # Save the expanded files info so we can set correct names in the task UIfor f in expanded_files:expanded_files_info.append(f)@@ -601,9 +428,6 @@ class ConnectorService:f"Original IDs: {file_ids}. This may indicate all IDs were folders "f"with no contents, or files that were filtered out.")- # If we have file_infos with download URLs, use original file_ids- # (OneDrive sharing IDs can't be expanded but can be downloaded directly)- # Exclude folders — they have no downloadable content on their own.if file_infos:non_folder_infos = [f for f in file_infos if not f.get("isFolder")]non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]@@ -619,10 +443,8 @@ class ConnectorService:except Exception as e:logger.error(f"Failed to expand file_ids via list_files(): {e}")- # Preserve intentional validation failures (e.g., folders-only selection)if isinstance(e, ValueError):raise- # Fallback path: still exclude known folders when metadata is availableif file_infos:non_folder_ids = [f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")@@ -630,11 +452,6 @@ class ConnectorService:expanded_file_ids = non_folder_ids or file_idselse:expanded_file_ids = file_ids- finally:- # Restore original config values- if hasattr(connector, "cfg"):- connector.cfg.file_ids = original_file_ids- connector.cfg.folder_ids = original_folder_ids# Create custom processor for specific connector filesfrom models.processors import ConnectorFileProcessordiff --git a/src/models/processors.py b/src/models/processors.pyindex 5f9970912..0cd6498fd 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -21,7 +21,7 @@ from utils.file_utils import ()from utils.hash_utils import hash_idfrom utils.logging_config import get_logger-from utils.opensearch_queries import build_filename_search_body, build_replace_filename_query+from utils.opensearch_queries import build_replace_filename_queryfrom .tasks import FileTask, TaskStatus, UploadTask@@ -30,6 +30,8 @@ logger = get_logger(__name__)DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"TEXT_PARSER_LABEL = "Text Parser"+DUPLICATE_FILENAME_WARNING = "A file with this name already exists."+if TYPE_CHECKING:from connectors.base import DocumentACL@@ -194,41 +196,27 @@ class TaskProcessor:(default ~1s), and the user-scoped client cannot force an``indices:admin/refresh`` (it lacks the privilege)."""+ from utils.opensearch_filenames import find_existing_filenames+max_retries = 3retry_delay = 1.0candidate_filenames = get_filename_aliases(filename)if not candidate_filenames:return False- # Keep track of aliases that still need checking across retries.- # If one alias was already checked successfully with no hits, we avoid- # re-querying it when another alias fails transiently.- pending_candidates = list(candidate_filenames)- # Retry strategy: only retry aliases that have not completed successfully.- # This avoids re-querying aliases already checked with no hits when a later- # alias fails transiently (e.g., timeout).for attempt in range(max_retries):try:- i = 0- while i < len(pending_candidates):- candidate = pending_candidates[i]- search_body = build_filename_search_body(candidate, size=1, source=False)- response = await opensearch_client.search(- index=get_index_name(), body=search_body- )- hits = response.get("hits", {}).get("hits", [])- if hits:- return True- # Successfully checked this alias with no hits; don't- # re-query it on future retries.- pending_candidates.pop(i)- continue- # All aliases checked with no hits. For post-ingest verification,- # the document may not be visible yet within the near-real-time- # refresh window — re-check every alias after a short delay.+ # One bulk existence check covering every alias — the shared+ # query semantic used by all duplicate-detection altitudes.+ if await find_existing_filenames(+ candidate_filenames, opensearch_client, get_index_name()+ ):+ return True+ # No alias exists. For post-ingest verification, the document+ # may not be visible yet within the near-real-time refresh+ # window — re-check after a short delay.if wait_for_visibility and attempt < max_retries - 1:- pending_candidates = list(candidate_filenames)await asyncio.sleep(retry_delay)retry_delay *= 2continue@@ -260,19 +248,84 @@ class TaskProcessor:retry_delay *= 2 # Exponential backoffreturn False+ async def resolve_duplicate_filename(+ self,+ filename: str,+ opensearch_client,+ *,+ replace: bool,+ owner_user_id: str | None,+ shared: bool = False,+ ) -> Literal["proceed", "skip", "replaced"]:+ """Single duplicate-filename policy shared by every processor.++ Checks whether a document with this filename (or one of its aliases)+ is already indexed and applies the caller's replace decision:++ * ``"proceed"`` — no duplicate; continue ingestion.+ * ``"skip"`` — duplicate and ``replace`` is False; the caller+ should finish via ``mark_duplicate_skipped``.+ * ``"replaced"`` — duplicate and ``replace`` is True; the existing+ chunks were deleted and the index refreshed, so+ ingestion can continue.+ """+ if not await self.check_filename_exists(filename, opensearch_client):+ return "proceed"+ if not replace:+ return "skip"++ logger.info(f"Replacing existing document: {filename}")+ deleted = await self.delete_document_by_filename(+ filename,+ opensearch_client,+ owner_user_id=owner_user_id,+ shared=shared,+ )+ if deleted == 0:+ logger.warning(+ "Replacement requested but deletion removed no chunks",+ filename=filename,+ )+ return "skip"+ # Refresh so the delete is visible before re-ingest. refresh is+ # index-wide (indices:admin/refresh) and cannot be DLS-scoped, so it+ # must run under the admin/service client, not the user client.+ try:+ await clients.opensearch.indices.refresh(index=get_index_name())+ except Exception as refresh_error:+ logger.warning(+ "Failed to refresh index after delete",+ error=str(refresh_error),+ )+ return "replaced"++ def mark_duplicate_skipped(self, upload_task: UploadTask, file_task: FileTask) -> None:+ """Uniform terminal state for a duplicate that was not replaced:+ SKIPPED, counted toward successful files, with a warning the task view+ surfaces. A declined replacement is a chosen outcome, not an error."""+ file_task.status = TaskStatus.SKIPPED+ file_task.error = None+ file_task.result = {+ "status": "skipped",+ "reason": "duplicate_filename",+ "warning": DUPLICATE_FILENAME_WARNING,+ }+ file_task.updated_at = time.time()+ upload_task.successful_files += 1+async def delete_document_by_filename(self,filename: str,opensearch_client,owner_user_id: str | None = None,shared: bool = False,- ) -> None:- """- Delete all chunks of a document with the given filename from OpenSearch.- """+ ) -> int:+ """Delete all chunks of a document with the given filename from+ OpenSearch. Returns the number of chunks deleted."""from config.settings import clients, get_index_namefrom utils.opensearch_delete import collect_visible_document_ids, delete_document_idsfrom utils.opensearch_queries import (+ build_anonymous_filename_query,build_owned_filename_query,build_replace_filename_query,)@@ -282,13 +335,20 @@ class TaskProcessor:if write_client is None:raise RuntimeError("Backend OpenSearch write client is unavailable")- deleted_count = 0if not owner_user_id:- logger.warning(- "Skipped delete_by_filename because owner_user_id is missing",- filename=filename,- )- return+ if shared:++ def build_query(fname, _owner):+ return build_anonymous_filename_query(fname)+ else:+ logger.warning(+ "Skipped delete_by_filename because owner_user_id is missing",+ filename=filename,+ )+ return 0++ else:+ build_query = build_replace_filename_query if shared else build_owned_filename_querycandidate_filenames = get_filename_aliases(filename)if not candidate_filenames:@@ -296,14 +356,9 @@ class TaskProcessor:"Skipped delete_by_filename because filename input is empty",filename=filename,)- return-- # When shared=True the document being replaced may have previously- # been ingested without an owner field (also shared), so the normal- # owner-scoped query would miss those chunks. Use a broader query- # that covers both owned and ownerless chunks for this filename.- build_query = build_replace_filename_query if shared else build_owned_filename_query+ return 0+ deleted_count = 0for candidate in candidate_filenames:document_ids = await collect_visible_document_ids(opensearch_client,@@ -318,6 +373,7 @@ class TaskProcessor:logger.info("Deleted existing document chunks", filename=filename, deleted_count=deleted_count)+ return deleted_countexcept Exception as e:logger.error("Failed to delete existing document", filename=filename, error=str(e))@@ -330,64 +386,31 @@ class TaskProcessor:owner_user_id: str,keep_filenames: list[str] | None = None,shared: bool = False,+ connector_type: str | None = None,) -> int:"""Delete indexed chunks for a connector file by its STABLE id.- Matches both ``connector_file_id`` (standard path, where ``document_id``- holds the content hash) and ``document_id`` (Langflow path, where it- holds the connector id). When ``keep_filenames`` is given, chunks whose- filename is one of those names are preserved — used to drop only the- stale OLD-name chunks left behind by a rename, since a connector file- keeps the same id across renames. Best-effort: logs and returns 0 on- failure so a cleanup miss never fails the task.+ Deletion semantics (dual-field id match, connector/owner/shared scoping,+ rename ``keep_filenames``) live in ``connectors.chunk_cleanup``. This+ wrapper is best-effort: logs and returns 0 on failure so a cleanup miss+ never fails the task.++ ``connector_type`` scopes the match to one connector type — the same+ value the chunks were indexed under — so an id that collides with a+ different connector's id can't take its chunks down with it."""- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids+ from connectors.chunk_cleanup import delete_connector_file_chunksif not file_id:return 0try:- write_client = clients.opensearch- if write_client is None:- raise RuntimeError("Backend OpenSearch write client is unavailable")-- owner_filter = (- {"bool": {"must_not": {"exists": {"field": "owner"}}}}- if shared- else {"term": {"owner": owner_user_id}}- )- query: dict[str, Any] = {- "bool": {- "filter": [- {- "bool": {- "should": [- {"term": {"document_id": file_id}},- {"term": {"connector_file_id": file_id}},- # Some deployments' indices predate this field's- # addition to the explicit mapping, so it was- # dynamically mapped as analyzed text with a- # `.keyword` multi-field instead of `keyword`.- {"term": {"connector_file_id.keyword": file_id}},- ],- "minimum_should_match": 1,- }- },- owner_filter,- ]- }- }- if keep_filenames:- query["bool"]["must_not"] = [{"terms": {"filename": keep_filenames}}]-- chunk_ids = await collect_visible_document_ids(+ return await delete_connector_file_chunks(+ [file_id],opensearch_client,- index=get_index_name(),- query=query,- )- return await delete_document_ids(- write_client,- index=get_index_name(),- document_ids=chunk_ids,+ connector_type=connector_type,+ owner_user_id=owner_user_id,+ shared=shared,+ keep_filenames=keep_filenames,)except Exception as e:logger.error(@@ -728,35 +751,15 @@ class DocumentFileProcessor(TaskProcessor):self.owner_user_id, self.jwt_token)- filename_exists = await self.check_filename_exists(original_filename, opensearch_client)-- if filename_exists and not self.replace_duplicates:- # Duplicate exists and user hasn't confirmed replacement- file_task.status = TaskStatus.FAILED- file_task.error = f"File with name '{original_filename}' already exists"- file_task.updated_at = time.time()- upload_task.failed_files += 1+ duplicate_action = await self.resolve_duplicate_filename(+ original_filename,+ opensearch_client,+ replace=self.replace_duplicates,+ owner_user_id=self.owner_user_id,+ )+ if duplicate_action == "skip":+ self.mark_duplicate_skipped(upload_task, file_task)return- elif filename_exists and self.replace_duplicates:- # Delete existing document before uploading new one- logger.info(f"Replacing existing document: {original_filename}")- await self.delete_document_by_filename(- original_filename,- opensearch_client,- owner_user_id=self.owner_user_id,- )- # Refresh index to make deletion visible before processing.- # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,- # so it must run under the admin/service client, not the user client.- from config.settings import get_index_name-- try:- await clients.opensearch.indices.refresh(index=get_index_name())- except Exception as refresh_error:- logger.warning(- "Failed to refresh index after delete",- error=str(refresh_error),- )# Compute hashfile_hash = hash_id(item)@@ -1007,7 +1010,11 @@ class ConnectorFileProcessor(TaskProcessor):))deleted_chunks = await self._delete_connector_chunks(- file_id, opensearch_client, self.user_id, shared=self.shared+ file_id,+ opensearch_client,+ self.user_id,+ shared=self.shared,+ connector_type=connector_type,)logger.warning(@@ -1055,6 +1062,18 @@ class ConnectorFileProcessor(TaskProcessor):self.user_id, self.jwt_token)+ duplicate_action = await self.resolve_duplicate_filename(+ file_task.filename,+ opensearch_client,+ replace=self.replace_duplicates,+ owner_user_id=self.user_id,+ shared=self.shared,+ )+ if duplicate_action == "skip":+ await self._reconcile_shared_owner(file_task.filename)+ self.mark_duplicate_skipped(upload_task, file_task)+ return+# Rename cleanup: a connector file keeps a stable id across renames,# but chunks are keyed by filename/content-hash, so a renamed file# leaves its OLD-name chunks orphaned. Drop chunks for this id whose@@ -1071,30 +1090,11 @@ class ConnectorFileProcessor(TaskProcessor):self.user_id,keep_filenames=get_filename_aliases(file_task.filename),shared=self.shared,+ connector_type=connector_type,)> 0)- if await self.check_filename_exists(file_task.filename, opensearch_client):- if not self.replace_duplicates:- await self._reconcile_shared_owner(file_task.filename)- file_task.status = TaskStatus.SKIPPED- file_task.error = None- file_task.result = {- "status": "skipped",- "reason": "duplicate_filename",- "warning": "A file with this name already exists.",- }- file_task.updated_at = time.time()- upload_task.successful_files += 1- return- await self.delete_document_by_filename(- file_task.filename,- opensearch_client,- owner_user_id=self.user_id,- shared=self.shared,- )-# Create temporary file from document contentsuffix = os.path.splitext(file_task.filename)[1]if not suffix:@@ -1353,6 +1353,7 @@ class S3FileProcessor(TaskProcessor):owner_email: str = None,models_service=None,docling_service=None,+ replace_duplicates: bool = False,):import boto3@@ -1367,6 +1368,7 @@ class S3FileProcessor(TaskProcessor):self.jwt_token = jwt_tokenself.owner_name = owner_nameself.owner_email = owner_email+ self.replace_duplicates = replace_duplicatesasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Download an S3 object and process it using DocumentService"""@@ -1378,6 +1380,21 @@ class S3FileProcessor(TaskProcessor):file_task.updated_at = time.time()try:+ # The S3 key doubles as the indexed filename, so the duplicate+ # gate can run before downloading the object.+ opensearch_client = self.document_service.session_manager.get_user_opensearch_client(+ self.owner_user_id, self.jwt_token+ )+ duplicate_action = await self.resolve_duplicate_filename(+ item,+ opensearch_client,+ replace=self.replace_duplicates,+ owner_user_id=self.owner_user_id,+ )+ if duplicate_action == "skip":+ self.mark_duplicate_skipped(upload_task, file_task)+ return+suffix = os.path.splitext(item)[1]with auto_cleanup_tempfile(suffix=suffix) as tmp_path:# Download object to temporary file@@ -1473,33 +1490,15 @@ class LangflowFileProcessor(TaskProcessor):self.owner_user_id, self.jwt_token)- filename_exists = await self.check_filename_exists(original_filename, opensearch_client)-- if filename_exists and not self.replace_duplicates:- # Duplicate exists and user hasn't confirmed replacement- file_task.status = TaskStatus.FAILED- file_task.error = f"File with name '{original_filename}' already exists"- file_task.updated_at = time.time()- upload_task.failed_files += 1+ duplicate_action = await self.resolve_duplicate_filename(+ original_filename,+ opensearch_client,+ replace=self.replace_duplicates,+ owner_user_id=self.owner_user_id,+ )+ if duplicate_action == "skip":+ self.mark_duplicate_skipped(upload_task, file_task)return- elif filename_exists and self.replace_duplicates:- # Delete existing document before uploading new one- logger.info(f"Replacing existing document: {original_filename}")- await self.delete_document_by_filename(- original_filename,- opensearch_client,- owner_user_id=self.owner_user_id,- )- # Refresh index to make deletion visible before processing.- # refresh is index-wide (indices:admin/refresh) and cannot be DLS-scoped,- # so it must run under the admin/service client, not the user client.- try:- await clients.opensearch.indices.refresh(index=get_index_name())- except Exception as refresh_error:- logger.warning(- "Failed to refresh index after delete",- error=str(refresh_error),- )# Read file content for processingwith open(item, "rb") as f:diff --git a/src/services/connector_access_service.py b/src/services/connector_access_service.pyindex a662fc1e8..613d5f8aa 100644--- a/src/services/connector_access_service.py+++ b/src/services/connector_access_service.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom sqlalchemy.ext.asyncio import AsyncSession-from connectors.registry import get_connector_classes+from connectors.registry import get_connector_class, get_connector_classesfrom db.repositories import WorkspaceConfigRepoCONNECTOR_ACCESS_SECTION = "connector_access"@@ -13,12 +13,12 @@ CONNECTOR_ACCESS_SECTION = "connector_access"# without touching this module.CONNECTOR_TYPES: tuple[str, ...] = tuple(cls.CONNECTOR_TYPE for cls in get_connector_classes())-_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos", "azure_blob"})-def is_bucket_connector_type(connector_type: str) -> bool:- """True for object-storage "bucket" connectors (S3, IBM COS, Azure Blob)."""- return connector_type in _BUCKET_CONNECTOR_TYPES+ """True for object-storage "bucket" connectors, per the class's+ CONNECTOR_KIND declaration (registry-derived, no hardcoded type list)."""+ cls = get_connector_class(connector_type)+ return cls is not None and cls.CONNECTOR_KIND == "bucket"def governable_connector_types() -> tuple[str, ...]:@@ -31,7 +31,7 @@ def governable_connector_types() -> tuple[str, ...]:if not is_azure_blob_enabled():types = tuple(t for t in types if t != "azure_blob")if is_cloud_context() and not IBM_AUTH_ENABLED:- types = tuple(t for t in types if t not in _BUCKET_CONNECTOR_TYPES)+ types = tuple(t for t in types if not is_bucket_connector_type(t))return typesdiff --git a/src/services/knowledge_filter_service.py b/src/services/knowledge_filter_service.pyindex 8937a041c..91cc5334f 100644--- a/src/services/knowledge_filter_service.py+++ b/src/services/knowledge_filter_service.py@@ -34,7 +34,7 @@ class KnowledgeFilterService:from config.settings import clients, get_index_namefrom utils.logging_config import get_logger- from utils.opensearch_queries import build_existing_filenames_agg_body+ from utils.opensearch_filenames import find_existing_filenamesdata_sources_by_filter: list[list[str] | None] = []all_filenames = set()@@ -58,13 +58,9 @@ class KnowledgeFilterService:if not all_filenames or clients.opensearch is None:return- existence_result = await clients.opensearch.search(- index=get_index_name(),- body=build_existing_filenames_agg_body(list(all_filenames)),+ existing_filenames = await find_existing_filenames(+ all_filenames, clients.opensearch, get_index_name())- existing_filenames = {- bucket["key"] for bucket in existence_result["aggregations"]["filenames"]["buckets"]- }for knowledge_filter, data_sources in zip(filters, data_sources_by_filter, strict=True):if data_sources:diff --git a/src/utils/opensearch_filenames.py b/src/utils/opensearch_filenames.pynew file mode 100644index 000000000..4f3b154a2--- /dev/null+++ b/src/utils/opensearch_filenames.py@@ -0,0 +1,52 @@+"""Shared filename-existence checks against the documents index.++Duplicate-filename decisions happen at several altitudes — the UI pre-check+endpoints, the API sync pre-filters, and the per-file processor backstop.+They must share one query semantic or the layers can disagree about what+counts as a duplicate. Every caller goes through this module: one terms+aggregation over exact ``filename`` keywords, with alias expansion via+``get_filename_aliases`` where a single logical filename is being checked.++The aggregation matters: collecting filenames from search *hits* undercounts+when a many-chunk document crowds other matches out of the hits window.+"""++from collections.abc import Iterable++from utils.file_utils import get_filename_aliases+from utils.opensearch_queries import build_existing_filenames_agg_body+++async def find_existing_filenames(+ candidates: Iterable[str],+ opensearch_client,+ index: str,+) -> set[str]:+ """Return the subset of exact candidate filenames that have at least one+ indexed chunk visible to this client.++ Callers own alias expansion (pass every variant that should count) and+ error handling — OpenSearch exceptions propagate.+ """+ unique = sorted({c for c in candidates if c})+ if not unique:+ return set()+ response = await opensearch_client.search(+ index=index,+ body=build_existing_filenames_agg_body(unique),+ )+ buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])+ return {bucket["key"] for bucket in buckets if bucket.get("key")}+++async def filename_exists(+ filename: str,+ opensearch_client,+ index: str,+) -> bool:+ """True when this filename — or any of its ingestion aliases — has indexed+ chunks visible to this client."""+ aliases = get_filename_aliases(filename)+ if not aliases:+ return False+ return bool(await find_existing_filenames(aliases, opensearch_client, index))diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.pydeleted file mode 100644index a292277ca..000000000--- a/tests/unit/api/test_delete_chunks_by_document_ids.py+++ /dev/null@@ -1,129 +0,0 @@-"""Unit tests for the bulk-delete helper used by the orphan-reconcile pass.--Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:-- empty input is a no-op (no OpenSearch call)-- non-empty input enumerates visible chunk _ids and deletes them one-by-one- by primary id (DLS-safe; `delete_by_query` is silently filtered under DLS)-- returns the count of successful single deletes-"""--import sys-from pathlib import Path-from unittest.mock import AsyncMock--import pytest--ROOT = Path(__file__).resolve().parent.parent.parent.parent-SRC = ROOT / "src"-if str(SRC) not in sys.path:- sys.path.insert(0, str(SRC))---async def test_empty_ids_short_circuits_without_calling_opensearch():- from api.documents import delete_chunks_by_document_ids-- opensearch_client = AsyncMock()- deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")-- assert deleted == 0- opensearch_client.search.assert_not_awaited()- opensearch_client.delete.assert_not_awaited()- opensearch_client.delete_by_query.assert_not_awaited()---async def test_deletes_each_visible_chunk_id_by_primary_id():- """The helper must enumerate visible chunk _ids and issue a primary-id- delete for each. delete_by_query is forbidden because DLS / certain- security plugins silently no-op it (returns deleted:N but leaves docs)."""- from api.documents import delete_chunks_by_document_ids-- opensearch_client = AsyncMock()- write_client = AsyncMock()- # Visible chunk _ids for the requested document_ids.- chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]- opensearch_client.search.return_value = {- "_scroll_id": None,- "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},- }- write_client.delete.return_value = {"result": "deleted"}-- deleted = await delete_chunks_by_document_ids(- ["doc-a", "doc-b"],- opensearch_client,- "test-index",- write_opensearch_client=write_client,- )-- assert deleted == len(chunk_ids)-- # delete_by_query must NOT be used — silently filtered under DLS.- opensearch_client.delete_by_query.assert_not_awaited()- write_client.delete_by_query.assert_not_awaited()-- # Search query must target the document_id field with terms(...).- search_call = opensearch_client.search.await_args- assert search_call.kwargs["index"] == "test-index"- assert search_call.kwargs["body"]["query"] == {"terms": {"document_id": ["doc-a", "doc-b"]}}-- # One primary-id delete per visible chunk, refresh=True so the delete is- # immediately visible to the re-index that typically follows.- opensearch_client.delete.assert_not_awaited()- delete_calls = write_client.delete.await_args_list- assert len(delete_calls) == len(chunk_ids)- for call, cid in zip(delete_calls, chunk_ids, strict=True):- assert call.kwargs["index"] == "test-index"- assert call.kwargs["id"] == cid- assert call.kwargs.get("refresh") is True---async def test_returns_zero_when_no_visible_chunks_match():- """When the search returns no hits, no deletes are issued and the count- is zero. This is the steady-state path (nothing to clean up)."""- from api.documents import delete_chunks_by_document_ids-- opensearch_client = AsyncMock()- write_client = AsyncMock()- opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}-- deleted = await delete_chunks_by_document_ids(- ["abc"],- opensearch_client,- "test-index",- write_opensearch_client=write_client,- )-- assert deleted == 0- opensearch_client.delete.assert_not_awaited()- write_client.delete.assert_not_awaited()---async def test_custom_field_parameter_is_used_in_query():- """Passing field='connector_file_id' must query that field and still delete- through the trusted backend write client."""- from api.documents import delete_chunks_by_document_ids-- opensearch_client = AsyncMock()- write_client = AsyncMock()- opensearch_client.search.return_value = {- "_scroll_id": None,- "hits": {"hits": [{"_id": "chunk-x"}]},- }- write_client.delete.return_value = {"result": "deleted"}-- deleted = await delete_chunks_by_document_ids(- ["src-file-1"],- opensearch_client,- "test-index",- write_opensearch_client=write_client,- field="connector_file_id",- )-- assert deleted == 1- search_call = opensearch_client.search.await_args- assert search_call.kwargs["body"]["query"] == {"terms": {"connector_file_id": ["src-file-1"]}}- opensearch_client.delete.assert_not_awaited()- write_client.delete.assert_awaited_once()diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.pyindex e0e982af1..3559f95bd 100644--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py@@ -6,7 +6,8 @@ The orphan-deletion safety net must:- preserve files present in any active connection,- enumerate visible chunks with the user-scoped client, then delete by primaryID with the trusted backend OpenSearch client,-- query either `document_id` or `connector_file_id` depending on the ingest path.+- match BOTH `document_id` and `connector_file_id` so orphan deletion covers+ chunks from either ingest layout (see connectors.chunk_cleanup)."""import json@@ -30,12 +31,13 @@ def _make_connection(connection_id: str, is_active: bool = True):def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):connector = MagicMock()connector.is_authenticated = authenticated+ response = {"files": [{"id": fid} for fid in remote_file_ids]}if raise_on_list:connector.list_files = AsyncMock(side_effect=RuntimeError("graph 503"))+ connector.list_selected_files = AsyncMock(side_effect=RuntimeError("graph 503"))else:- connector.list_files = AsyncMock(- return_value={"files": [{"id": fid} for fid in remote_file_ids]}- )+ connector.list_files = AsyncMock(return_value=response)+ connector.list_selected_files = AsyncMock(return_value=response)return connector@@ -222,13 +224,75 @@ async def test_happy_path_deletes_orphans(monkeypatch):opensearch_client.delete.assert_not_awaited()search_body = opensearch_client.search.await_args.kwargs["body"]- assert search_body["query"] == {"terms": {"document_id": ["b"]}}+ shoulds = search_body["query"]["bool"]["filter"][0]["bool"]["should"]+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}+ assert fields == {+ "document_id": ["b"],+ "connector_file_id": ["b"],+ "connector_file_id.keyword": ["b"],+ }assert [call.kwargs["id"] for call in write_client.delete.await_args_list] == ["chunk-b-1","chunk-b-2",]+async def test_happy_path_private_scopes_to_owner(monkeypatch):+ """Private orphan cleanup must filter by connector_type and owner."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])+ _patch_write_client(monkeypatch)+ sm = _make_session_manager(opensearch_client)++ await reconcile_orphans_for_connector_type(+ connector_type="google_drive",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ search_body = opensearch_client.search.await_args.kwargs["body"]+ filters = search_body["query"]["bool"]["filter"]+ assert {"term": {"connector_type": "google_drive"}} in filters+ assert {"term": {"owner": "alice"}} in filters+++async def test_happy_path_shared_scopes_to_ownerless(monkeypatch):+ """Shared orphan cleanup must filter by connector_type and ownerless chunks."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])+ _patch_write_client(monkeypatch)+ sm = _make_session_manager(opensearch_client)++ await reconcile_orphans_for_connector_type(+ connector_type="ibm_cos",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ shared=True,+ )++ search_body = opensearch_client.search.await_args.kwargs["body"]+ filters = search_body["query"]["bool"]["filter"]+ assert {"term": {"connector_type": "ibm_cos"}} in filters+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters+ assert {"term": {"owner": "alice"}} not in filters++@pytest.mark.asyncioasync def test_delete_failure_does_not_raise(monkeypatch):from api.connectors import reconcile_orphans_for_connector_type@@ -310,6 +374,8 @@ async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():@pytest.mark.asyncioasync def test_paginated_listing_aggregates_all_pages():+ """Connectors without cfg (e.g. bucket connectors) use the paginated+ list_files() path. Verify that all pages are consumed."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")@@ -320,6 +386,7 @@ async def test_paginated_listing_aggregates_all_pages():{"files": [{"id": "b"}, {"id": "c"}]},]connector.list_files = AsyncMock(side_effect=pages)+ del connector.cfgservice = _make_service([conn], connector_lookup={"c1": connector})opensearch_client = _make_opensearch_client()@@ -340,7 +407,11 @@ async def test_paginated_listing_aggregates_all_pages():@pytest.mark.asyncio-async def test_connector_file_id_field_used_when_specified(monkeypatch):+async def test_orphan_delete_matches_both_id_layouts(monkeypatch):+ """Orphan deletion must cover chunks from either ingest layout: the+ standard path keys chunks by connector_file_id (document_id is a content+ hash) while the Langflow path keys them by document_id. A single-field+ query would miss one of them."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")@@ -357,42 +428,21 @@ async def test_connector_file_id_field_used_when_specified(monkeypatch):session_manager=sm,jwt_token=None,existing_file_ids=["sp-guid-a", "sp-guid-b"],- id_field="connector_file_id",)assert result == ["sp-guid-b"]search_body = opensearch_client.search.await_args.kwargs["body"]- assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}+ shoulds = search_body["query"]["bool"]["filter"][0]["bool"]["should"]+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}+ assert fields == {+ "document_id": ["sp-guid-b"],+ "connector_file_id": ["sp-guid-b"],+ "connector_file_id.keyword": ["sp-guid-b"],+ }assert write_client.delete.await_count == 2opensearch_client.delete.assert_not_awaited()-async def test_document_id_field_used_by_default(monkeypatch):- from api.connectors import reconcile_orphans_for_connector_type-- conn = _make_connection("c1")- connector = _make_connector(remote_file_ids=["lf-id-a"])- service = _make_service([conn], connector_lookup={"c1": connector})- opensearch_client = _make_opensearch_client(chunk_ids=["chunk-lf-b-0"])- write_client = _patch_write_client(monkeypatch)- sm = _make_session_manager(opensearch_client)-- result = await reconcile_orphans_for_connector_type(- connector_type="sharepoint",- user_id="alice",- connector_service=service,- session_manager=sm,- jwt_token=None,- existing_file_ids=["lf-id-a", "lf-id-b"],- )-- assert result == ["lf-id-b"]- search_body = opensearch_client.search.await_args.kwargs["body"]- assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}- write_client.delete.assert_awaited_once()--@pytest.mark.asyncioasync def test_preview_subtracts_orphans_from_resync_count(monkeypatch):from api.connectors import _preview_orphans_for_connector_typediff --git a/tests/unit/api/test_sync_existing_connector_files.py b/tests/unit/api/test_sync_existing_connector_files.pynew file mode 100644index 000000000..a9718a4f8--- /dev/null+++ b/tests/unit/api/test_sync_existing_connector_files.py@@ -0,0 +1,144 @@+"""Outcome contract of _sync_existing_connector_files — the shared+no-selection Sync flow behind both connector_sync and sync_all_connectors."""++from unittest.mock import AsyncMock, MagicMock++import pytest++from api import connectors as connectors_api+++def _make_services():+ connector_service = MagicMock()+ connector_service.sync_specific_files = AsyncMock(return_value="task-1")+ connector_service.sync_connector_files = AsyncMock(return_value="task-2")+ connector_service.get_connector = AsyncMock(return_value=MagicMock())+ working_connection = MagicMock()+ working_connection.connection_id = "conn-1"+ return connector_service, working_connection+++async def _run(+ connector_service,+ working_connection,+ *,+ connector_type="google_drive",+ existing_file_ids=None,+ existing_filenames=None,+ **kwargs,+):+ return await connectors_api._sync_existing_connector_files(+ connector_type=connector_type,+ working_connection=working_connection,+ user_id="user-1",+ connector_service=connector_service,+ session_manager=MagicMock(),+ jwt_token="jwt",+ existing_file_ids=existing_file_ids or [],+ existing_filenames=existing_filenames or [],+ id_field="connector_file_id",+ **kwargs,+ )+++async def test_oauth_sync_uses_connector_replace_capability(monkeypatch):+ monkeypatch.setattr(+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])+ )+ connector_service, working_connection = _make_services()++ result = await _run(connector_service, working_connection, existing_file_ids=["f1", "f2"])++ assert result == {"outcome": "synced", "task_id": "task-1"}+ call = connector_service.sync_specific_files.await_args+ assert call.args[2] == ["f1", "f2"]+ # google_drive declares SYNC_REPLACES_DUPLICATES=True.+ assert call.kwargs["replace_duplicates"] is True+++async def test_deleted_only_when_orphan_reconcile_removes_everything(monkeypatch):+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=["f1", "f2"]),+ )+ connector_service, working_connection = _make_services()++ result = await _run(connector_service, working_connection, existing_file_ids=["f1", "f2"])++ assert result == {"outcome": "deleted_only"}+ connector_service.sync_specific_files.assert_not_awaited()+++async def test_reconcile_false_skips_orphan_pass(monkeypatch):+ reconcile = AsyncMock(return_value=["f1"])+ monkeypatch.setattr(connectors_api, "reconcile_orphans_for_connector_type", reconcile)+ connector_service, working_connection = _make_services()++ result = await _run(+ connector_service,+ working_connection,+ existing_file_ids=["f1"],+ reconcile=False,+ )++ assert result["outcome"] == "synced"+ reconcile.assert_not_awaited()+++async def test_timestamp_connector_up_to_date_when_no_changes(monkeypatch):+ monkeypatch.setattr(+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])+ )+ monkeypatch.setattr(connectors_api, "bucket_changed_file_ids", AsyncMock(return_value=[]))+ connector_service, working_connection = _make_services()++ result = await _run(+ connector_service,+ working_connection,+ connector_type="aws_s3",+ existing_file_ids=["k1"],+ )++ assert result == {"outcome": "up_to_date"}+ connector_service.sync_specific_files.assert_not_awaited()+++async def test_timestamp_connector_syncs_only_changed_ids_with_replace(monkeypatch):+ monkeypatch.setattr(+ connectors_api, "reconcile_orphans_for_connector_type", AsyncMock(return_value=[])+ )+ monkeypatch.setattr(connectors_api, "bucket_changed_file_ids", AsyncMock(return_value=["k2"]))+ connector_service, working_connection = _make_services()++ result = await _run(+ connector_service,+ working_connection,+ connector_type="aws_s3",+ existing_file_ids=["k1", "k2"],+ )++ assert result == {"outcome": "synced", "task_id": "task-1"}+ call = connector_service.sync_specific_files.await_args+ assert call.args[2] == ["k2"]+ assert call.kwargs["replace_duplicates"] is True+++async def test_filename_fallback_when_no_ids(monkeypatch):+ connector_service, working_connection = _make_services()++ result = await _run(+ connector_service,+ working_connection,+ existing_filenames=["a.pdf", "b.pdf"],+ )++ assert result == {"outcome": "synced", "task_id": "task-2"}+ call = connector_service.sync_connector_files.await_args+ assert call.kwargs["filename_filter"] == {"a.pdf", "b.pdf"}diff --git a/tests/unit/connectors/test_change_detection_capability.py b/tests/unit/connectors/test_change_detection_capability.pynew file mode 100644index 000000000..3e61d23e7--- /dev/null+++ b/tests/unit/connectors/test_change_detection_capability.py@@ -0,0 +1,49 @@+"""Change-detection strategy is a connector capability, not a hardcoded list.++``_connector_uses_timestamp_change_detection`` reads ``CHANGE_DETECTION`` from+the registered connector class, and ``is_bucket_connector_type`` derives from+``CONNECTOR_KIND`` — neither consults a type-string set anymore.+"""++import pytest++from api.connectors import _connector_uses_timestamp_change_detection+from connectors.aws_s3 import S3Connector+from connectors.base import BaseConnector+from services.connector_access_service import is_bucket_connector_type+++def test_base_default_is_replace_always():+ assert BaseConnector.CHANGE_DETECTION == "replace_always"+++def test_s3_declares_timestamp_change_detection():+ assert S3Connector.CHANGE_DETECTION == "timestamp"+ assert _connector_uses_timestamp_change_detection("aws_s3") is True++[email protected]("connector_type", ["google_drive", "sharepoint", "onedrive"])+def test_oauth_connectors_use_replace_always(connector_type):+ assert _connector_uses_timestamp_change_detection(connector_type) is False+++def test_unknown_type_defaults_to_replace_always():+ assert _connector_uses_timestamp_change_detection("no_such_connector") is False+++def test_capability_read_from_registered_class(monkeypatch):+ class TimestampConnector:+ CONNECTOR_TYPE = "custom_ts"+ CHANGE_DETECTION = "timestamp"++ monkeypatch.setattr(+ "connectors.registry.get_connector_class",+ lambda connector_type: TimestampConnector if connector_type == "custom_ts" else None,+ )+ assert _connector_uses_timestamp_change_detection("custom_ts") is True+++def test_is_bucket_connector_type_derived_from_kind():+ assert is_bucket_connector_type("aws_s3") is True+ assert is_bucket_connector_type("google_drive") is False+ assert is_bucket_connector_type("no_such_connector") is Falsediff --git a/tests/unit/connectors/test_chunk_cleanup.py b/tests/unit/connectors/test_chunk_cleanup.pynew file mode 100644index 000000000..ff7370a05--- /dev/null+++ b/tests/unit/connectors/test_chunk_cleanup.py@@ -0,0 +1,165 @@+"""Contract of connectors.chunk_cleanup — the single deletion path for a+connector file's indexed chunks (orphan reconcile, 404 cleanup, rename+cleanup all go through it).++- empty input is a no-op (no OpenSearch call)+- the query matches BOTH id layouts (document_id and connector_file_id)+- DLS-safe: visible chunk _ids are enumerated with the caller's client, then+ deleted one-by-one by primary id with the trusted backend write client+ (`delete_by_query` is silently filtered under DLS)+- owner/shared scoping and rename keep_filenames shape the query+"""++from unittest.mock import AsyncMock++import pytest++from connectors.chunk_cleanup import (+ build_connector_file_chunks_query,+ delete_connector_file_chunks,+)+++def write_client(monkeypatch):+ import config.settings as cfg++ client = AsyncMock()+ client.delete = AsyncMock(return_value={"result": "deleted"})+ monkeypatch.setattr(cfg.clients, "opensearch", client)+ return client+++def _id_terms(query) -> dict:+ shoulds = query["bool"]["filter"][0]["bool"]["should"]+ return {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}+++def test_query_matches_both_id_fields():+ query = build_connector_file_chunks_query(["file-1", "file-2"])+ fields = _id_terms(query)+ assert fields == {+ "document_id": ["file-1", "file-2"],+ "connector_file_id": ["file-1", "file-2"],+ "connector_file_id.keyword": ["file-1", "file-2"],+ }+ assert query["bool"]["filter"][0]["bool"]["minimum_should_match"] == 1+++def test_query_owner_scoping():+ query = build_connector_file_chunks_query(["f"], owner_user_id="alice")+ assert {"term": {"owner": "alice"}} in query["bool"]["filter"]+++def test_query_shared_targets_ownerless_chunks():+ query = build_connector_file_chunks_query(["f"], owner_user_id="alice", shared=True)+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in query["bool"]["filter"]+ assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]+++def test_query_keep_filenames_excluded():+ query = build_connector_file_chunks_query(["f"], keep_filenames=["new.pdf", "new_.pdf"])+ assert query["bool"]["must_not"] == [{"terms": {"filename": ["new.pdf", "new_.pdf"]}}]+++def test_query_unscoped_when_no_owner():+ query = build_connector_file_chunks_query(["f"])+ assert len(query["bool"]["filter"]) == 1+ assert "must_not" not in query["bool"]+++def test_query_connector_type_filter():+ query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos")+ assert {"term": {"connector_type": "ibm_cos"}} in query["bool"]["filter"]+++def test_query_connector_type_with_private_owner():+ query = build_connector_file_chunks_query(+ ["f"], connector_type="google_drive", owner_user_id="alice"+ )+ filters = query["bool"]["filter"]+ assert {"term": {"connector_type": "google_drive"}} in filters+ assert {"term": {"owner": "alice"}} in filters+++def test_query_connector_type_with_shared_scope():+ query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos", shared=True)+ filters = query["bool"]["filter"]+ assert {"term": {"connector_type": "ibm_cos"}} in filters+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters+ assert {"term": {"owner": "alice"}} not in filters+++def test_query_no_connector_type_when_none():+ query = build_connector_file_chunks_query(["f"])+ for f in query["bool"]["filter"]:+ if "term" in f:+ assert "connector_type" not in f["term"]+++async def test_empty_ids_short_circuit_without_calling_opensearch(write_client):+ opensearch_client = AsyncMock()++ deleted = await delete_connector_file_chunks([], opensearch_client)++ assert deleted == 0+ opensearch_client.search.assert_not_awaited()+ write_client.delete.assert_not_awaited()+++async def test_deletes_each_visible_chunk_id_by_primary_id(write_client, monkeypatch):+ """Visible chunk _ids are enumerated with the caller's client and deleted+ one-by-one via the backend write client. delete_by_query is forbidden —+ DLS / certain security plugins silently no-op it."""+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")++ opensearch_client = AsyncMock()+ chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]+ opensearch_client.search.return_value = {+ "_scroll_id": None,+ "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},+ }++ deleted = await delete_connector_file_chunks(+ ["doc-a", "doc-b"], opensearch_client, refresh=True+ )++ assert deleted == len(chunk_ids)+ opensearch_client.delete_by_query.assert_not_awaited()+ write_client.delete_by_query.assert_not_awaited()++ search_call = opensearch_client.search.await_args+ assert search_call.kwargs["index"] == "test-index"+ fields = _id_terms(search_call.kwargs["body"]["query"])+ assert fields["document_id"] == ["doc-a", "doc-b"]+ assert fields["connector_file_id"] == ["doc-a", "doc-b"]+ assert fields["connector_file_id.keyword"] == ["doc-a", "doc-b"]++ opensearch_client.delete.assert_not_awaited()+ delete_calls = write_client.delete.await_args_list+ assert [call.kwargs["id"] for call in delete_calls] == chunk_ids+ for call in delete_calls:+ assert call.kwargs["index"] == "test-index"+ assert call.kwargs.get("refresh") is True+++async def test_returns_zero_when_no_visible_chunks_match(write_client):+ opensearch_client = AsyncMock()+ opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}++ deleted = await delete_connector_file_chunks(["abc"], opensearch_client)++ assert deleted == 0+ write_client.delete.assert_not_awaited()+++async def test_raises_when_write_client_unavailable(monkeypatch):+ import config.settings as cfg++ monkeypatch.setattr(cfg.clients, "opensearch", None)+ with pytest.raises(RuntimeError, match="write client"):+ await delete_connector_file_chunks(["abc"], AsyncMock())diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.pyindex 8d0d98246..fc74f3632 100644--- a/tests/unit/connectors/test_connector_file_type_validation.py+++ b/tests/unit/connectors/test_connector_file_type_validation.py@@ -27,14 +27,14 @@ async def test_sync_specific_files_does_not_raise_on_incompatible_type():connector.is_authenticated = True# Mock list_files returning an incompatible file (e.g. an .exe)- connector.list_files = AsyncMock(- return_value={- "files": [- {"id": "file-1", "name": "document.pdf"},- {"id": "file-2", "name": "program.exe"},- ]- }- )+ expanded_response = {+ "files": [+ {"id": "file-1", "name": "document.pdf"},+ {"id": "file-2", "name": "program.exe"},+ ]+ }+ connector.list_files = AsyncMock(return_value=expanded_response)+ connector.list_selected_files = AsyncMock(return_value=expanded_response)connector.cfg = MagicMock()service.get_connector = AsyncMock(return_value=connector)@@ -104,18 +104,18 @@ async def test_connector_check_duplicates():connector.authenticate = AsyncMock(return_value=True)# Mock folder expansion- connector.list_files = AsyncMock(- return_value={- "files": [- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},- {- "id": "file-2",- "name": "new_file.docx",- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- },- ]- }- )+ expanded_files = {+ "files": [+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},+ {+ "id": "file-2",+ "name": "new_file.docx",+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ },+ ]+ }+ connector.list_files = AsyncMock(return_value=expanded_files)+ connector.list_selected_files = AsyncMock(return_value=expanded_files)connector.cfg = MagicMock()connector_service.get_connector = AsyncMock(return_value=connector)@@ -126,7 +126,9 @@ async def test_connector_check_duplicates():# Mock search return value: existing.pdf exists, new_file.docx does notopensearch_client.search = AsyncMock(- return_value={"hits": {"hits": [{"_source": {"filename": "existing.pdf"}}]}}+ return_value={+ "aggregations": {"filenames": {"buckets": [{"key": "existing.pdf", "doc_count": 1}]}}+ })user = MagicMock()@@ -286,18 +288,18 @@ async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selectedconnector.is_authenticated = Trueconnector.authenticate = AsyncMock(return_value=True)connector.cfg = MagicMock()- connector.list_files = AsyncMock(- return_value={- "files": [- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},- {- "id": "file-2",- "name": "existing.docx",- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- },- ]- }- )+ expanded_files = {+ "files": [+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},+ {+ "id": "file-2",+ "name": "existing.docx",+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ },+ ]+ }+ connector.list_files = AsyncMock(return_value=expanded_files)+ connector.list_selected_files = AsyncMock(return_value=expanded_files)connector_service.get_connector = AsyncMock(return_value=connector)connector_service.sync_specific_files = AsyncMock(return_value="task-id")@@ -306,11 +308,13 @@ async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selectedsession_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)opensearch_client.search = AsyncMock(return_value={- "hits": {- "hits": [- {"_source": {"filename": "existing.pdf"}},- {"_source": {"filename": "existing.docx"}},- ]+ "aggregations": {+ "filenames": {+ "buckets": [+ {"key": "existing.pdf", "doc_count": 1},+ {"key": "existing.docx", "doc_count": 1},+ ]+ }}})@@ -356,18 +360,18 @@ async def test_connector_sync_skip_duplicates_submits_only_expanded_non_duplicatconnector.is_authenticated = Trueconnector.authenticate = AsyncMock(return_value=True)connector.cfg = MagicMock()- connector.list_files = AsyncMock(- return_value={- "files": [- {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},- {- "id": "file-2",- "name": "new_file.docx",- "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- },- ]- }- )+ expanded_files = {+ "files": [+ {"id": "file-1", "name": "existing.pdf", "mimeType": "application/pdf"},+ {+ "id": "file-2",+ "name": "new_file.docx",+ "mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ },+ ]+ }+ connector.list_files = AsyncMock(return_value=expanded_files)+ connector.list_selected_files = AsyncMock(return_value=expanded_files)connector_service.get_connector = AsyncMock(return_value=connector)connector_service.sync_specific_files = AsyncMock(return_value="task-id")@@ -375,7 +379,9 @@ async def test_connector_sync_skip_duplicates_submits_only_expanded_non_duplicatopensearch_client = AsyncMock()session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)opensearch_client.search = AsyncMock(- return_value={"hits": {"hits": [{"_source": {"filename": "existing.pdf"}}]}}+ return_value={+ "aggregations": {"filenames": {"buckets": [{"key": "existing.pdf", "doc_count": 1}]}}+ })response = await connectors_api.connector_sync(diff --git a/tests/unit/connectors/test_langflow_connector_txt_to_md.py b/tests/unit/connectors/test_langflow_connector_txt_to_md.pydeleted file mode 100644index 4ecc23d53..000000000--- a/tests/unit/connectors/test_langflow_connector_txt_to_md.py+++ /dev/null@@ -1,115 +0,0 @@-"""Regression guard for SharePoint .txt ingestion via the Langflow connector-path.--Background: Langflow's docling component fails on text/plain. The-user-upload path already applies a .txt -> .md rename (commit f6b9fe0).-This test pins that the connector path-(`ConnectorService.process_connector_document`) applies the same rule, so-a SharePoint .txt reaches Langflow as .md / text/markdown — not-the unhandled text/plain shape that previously broke ingestion.--This test drives the production code path; if the rename is ever removed-or shorted around, the assertions will catch it.-"""--import sys-from datetime import datetime-from pathlib import Path-from unittest.mock import AsyncMock, MagicMock--import pytest--ROOT = Path(__file__).resolve().parent.parent.parent.parent-SRC = ROOT / "src"-if str(SRC) not in sys.path:- sys.path.insert(0, str(SRC))---def _make_service():- """Build a ConnectorService with the surface- process_connector_document touches stubbed.- """- from connectors.service import ConnectorService-- service = ConnectorService.__new__(ConnectorService)-- # Mock Langflow service- langflow_service = MagicMock()- langflow_service.upload_and_ingest_file = AsyncMock(return_value={"status": "ok"})- langflow_service.merge_ui_ingest_settings_into_tweaks = MagicMock(return_value={})- service.langflow_service = langflow_service- service.task_service = None- # _get_effective_sync_jwt passes jwt_token through when no session manager- service.session_manager = None-- return service, langflow_service---def _make_document(filename: str, mimetype: str, content: bytes = b"hello world"):- from connectors.base import ConnectorDocument, DocumentACL-- return ConnectorDocument(- id="graph-item-id-stable",- filename=filename,- mimetype=mimetype,- content=content,- source_url="https://contoso.sharepoint.com/.../notes.txt",- acl=DocumentACL(owner="alice"),- modified_time=datetime(2026, 5, 21),- created_time=datetime(2026, 5, 1),- metadata={"site": "marketing"},- )---async def test_sharepoint_txt_is_uploaded_to_langflow_as_md():- """A connector document with mimetype text/plain must reach Langflow's- upload_and_ingest_file with a .md filename and text/markdown mimetype."""- service, langflow_service = _make_service()- document = _make_document(filename="notes.txt", mimetype="text/plain")-- await service.process_connector_document(- document=document,- owner_user_id="alice",- connector_type="sharepoint",- jwt_token="jwt",- )-- langflow_service.upload_and_ingest_file.assert_awaited_once()- kwargs = langflow_service.upload_and_ingest_file.await_args.kwargs- file_tuple = kwargs.get("file_tuple")- filename, content, mimetype = file_tuple-- assert filename.endswith(".md"), (- f"SharePoint .txt must reach Langflow as .md; got {filename!r}. "- "Langflow's docling component fails on text/plain — see "- "langflow_safe_filename_and_mimetype."- )- assert mimetype == "text/markdown", (- f"SharePoint .txt must reach Langflow as text/markdown; got {mimetype!r}."- )- # Content bytes are unchanged by the rename.- assert content == document.content---async def test_sharepoint_pdf_passes_through_untouched():- """The rename rule must NOT touch non-.txt files."""- service, langflow_service = _make_service()- document = _make_document(- filename="report.pdf", mimetype="application/pdf", content=b"%PDF-1.4..."- )-- await service.process_connector_document(- document=document,- owner_user_id="alice",- connector_type="sharepoint",- jwt_token="jwt",- )-- langflow_service.upload_and_ingest_file.assert_awaited_once()- kwargs = langflow_service.upload_and_ingest_file.await_args.kwargs- file_tuple = kwargs.get("file_tuple")- filename, _content, mimetype = file_tuple- assert filename == "report.pdf"- assert mimetype == "application/pdf"diff --git a/tests/unit/connectors/test_list_selected_files.py b/tests/unit/connectors/test_list_selected_files.pynew file mode 100644index 000000000..f74e0501b--- /dev/null+++ b/tests/unit/connectors/test_list_selected_files.py@@ -0,0 +1,115 @@+"""BaseConnector.list_selected_files() — owns the cfg save/restore so+callers don't have to manually back up and restore file_ids/folder_ids."""++from dataclasses import dataclass++import pytest++from connectors.base import BaseConnector+++@dataclass+class _FakeCfg:+ file_ids: list[str] | None = None+ folder_ids: list[str] | None = None+++class _FakeConnector(BaseConnector):+ CONNECTOR_TYPE = "fake"++ def __init__(self):+ self.cfg = _FakeCfg(file_ids=["orig-1"], folder_ids=["folder-orig"])++ async def authenticate(self):+ return True++ async def setup_subscription(self):+ return ""++ async def list_files(self, page_token=None, max_files=None, **kwargs):+ return {"files": [{"id": fid} for fid in (self.cfg.file_ids or [])]}++ async def get_file_content(self, file_id):+ raise NotImplementedError++ async def handle_webhook(self, payload):+ return []++ async def cleanup_subscription(self, subscription_id):+ return True+++async def test_restores_cfg_after_success():+ conn = _FakeConnector()+ result = await conn.list_selected_files(["a", "b"])+ assert [f["id"] for f in result["files"]] == ["a", "b"]+ assert conn.cfg.file_ids == ["orig-1"]+ assert conn.cfg.folder_ids == ["folder-orig"]+++async def test_restores_cfg_after_exception():+ class _Broken(_FakeConnector):+ async def list_files(self, **kwargs):+ raise RuntimeError("boom")++ conn = _Broken()+ with pytest.raises(RuntimeError, match="boom"):+ await conn.list_selected_files(["x"])+ assert conn.cfg.file_ids == ["orig-1"]+ assert conn.cfg.folder_ids == ["folder-orig"]+++async def test_folder_ids_cleared_by_default():+ """list_selected_files sets folder_ids to None during the call."""+ observed_folder_ids = None++ class _Capture(_FakeConnector):+ async def list_files(self, **kwargs):+ nonlocal observed_folder_ids+ observed_folder_ids = self.cfg.folder_ids+ return {"files": []}++ conn = _Capture()+ await conn.list_selected_files(["a"])+ assert observed_folder_ids is None+++async def test_raises_when_cfg_is_none():+ """cfg-less (bucket) connectors must not silently fall back to list_files().++ BaseConnector declares cfg=None as a class default, so a bucket connector+ reaches list_selected_files with cfg None. Falling back to list_files()+ there would list the connector's entire account and ignore file_ids —+ exactly the "all buckets ingested" regression. Callers must gate on+ `cfg is not None` instead, so this path is a loud error, not a silent+ whole-account listing.+ """++ class _NoCfg(_FakeConnector):+ def __init__(self):+ self.cfg = None++ conn = _NoCfg()+ with pytest.raises(NotImplementedError):+ await conn.list_selected_files(["a"])+++async def test_folder_ids_passed_through():+ observed = {}++ class _Capture(_FakeConnector):+ async def list_files(self, **kwargs):+ nonlocal observed+ observed = {"file_ids": self.cfg.file_ids, "folder_ids": self.cfg.folder_ids}+ return {"files": []}++ conn = _Capture()+ await conn.list_selected_files(["f1"], folder_ids=["fold-1"])+ assert observed == {"file_ids": ["f1"], "folder_ids": ["fold-1"]}+ assert conn.cfg.file_ids == ["orig-1"]+ assert conn.cfg.folder_ids == ["folder-orig"]diff --git a/tests/unit/connectors/test_sync_replace_capability.py b/tests/unit/connectors/test_sync_replace_capability.pynew file mode 100644index 000000000..162e439e5--- /dev/null+++ b/tests/unit/connectors/test_sync_replace_capability.py@@ -0,0 +1,57 @@+"""Sync replace-on-duplicate is a connector capability, not a hardcoded list.++``_connector_sync_should_replace`` reads ``SYNC_REPLACES_DUPLICATES`` from the+connector class registered for the type. The BaseConnector default is True so+a new connector gets change-propagating sync without touching the API module.+"""++import pytest++from api.connectors import _connector_sync_should_replace+from connectors.base import BaseConnector++[email protected]("connector_type", ["google_drive", "sharepoint", "onedrive"])+def test_builtin_oauth_connectors_replace_on_sync(connector_type):+ assert _connector_sync_should_replace(connector_type) is True+++def test_unknown_connector_type_does_not_replace():+ assert _connector_sync_should_replace("no_such_connector") is False+++def test_new_connector_inherits_replace_default():+ class NewConnector(BaseConnector):+ CONNECTOR_TYPE = "new_connector"++ async def authenticate(self): # pragma: no cover - capability test only+ return True++ async def setup_subscription(self): # pragma: no cover+ raise NotImplementedError++ async def list_files(self, page_token=None, max_files=None, **kwargs): # pragma: no cover+ raise NotImplementedError++ async def get_file_content(self, file_id): # pragma: no cover+ raise NotImplementedError++ async def handle_webhook(self, payload): # pragma: no cover+ raise NotImplementedError++ async def cleanup_subscription(self, subscription_id): # pragma: no cover+ raise NotImplementedError++ assert NewConnector.SYNC_REPLACES_DUPLICATES is True+++def test_connector_can_opt_out_via_attribute(monkeypatch):+ class OptOutConnector:+ CONNECTOR_TYPE = "opt_out"+ SYNC_REPLACES_DUPLICATES = False++ monkeypatch.setattr(+ "connectors.registry.get_connector_class",+ lambda connector_type: OptOutConnector if connector_type == "opt_out" else None,+ )+ assert _connector_sync_should_replace("opt_out") is Falsediff --git a/tests/unit/connectors/test_sync_specific_files_deleted_ids.py b/tests/unit/connectors/test_sync_specific_files_deleted_ids.pyindex 29936d45d..7be77eaab 100644--- a/tests/unit/connectors/test_sync_specific_files_deleted_ids.py+++ b/tests/unit/connectors/test_sync_specific_files_deleted_ids.py@@ -35,7 +35,9 @@ def _make_service(live_files: list[dict]):connector = MagicMock()connector.is_authenticated = Trueconnector.cfg = MagicMock()- connector.list_files = AsyncMock(return_value={"files": live_files})+ response = {"files": live_files}+ connector.list_files = AsyncMock(return_value=response)+ connector.list_selected_files = AsyncMock(return_value=response)service.get_connector = AsyncMock(return_value=connector)return servicediff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex a40bffd8a..84423de08 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -49,6 +49,13 @@ def _make_search_response(has_hit: bool) -> dict:return {"hits": {"hits": [{"_id": "x"}] if has_hit else []}}+def _make_filename_agg_response(has_hit: bool, filename: str = "My Report.pdf") -> dict:+ """The filename duplicate check runs a terms aggregation (see+ utils/opensearch_filenames), not a hits query."""+ buckets = [{"key": filename, "doc_count": 1}] if has_hit else []+ return {"aggregations": {"filenames": {"buckets": buckets}}}++def _make_upload_task() -> UploadTask:return UploadTask(task_id="task-1", total_files=1)@@ -94,6 +101,10 @@ def _wire_connector_processor(return _make_search_response(rename_stale_exists)if "document_id" in query_str:return _make_search_response(hash_exists)+ if isinstance(body, dict) and "aggs" in body:+ # Bulk duplicate check (terms aggregation over filename aliases).+ return _make_filename_agg_response(filename_exists)+ # Hits-based filename scroll (delete_document_by_filename id collection).return _make_search_response(filename_exists)opensearch_client.search = mock_search@@ -220,9 +231,14 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404(search_call = opensearch_client.search.await_argsquery = search_call.kwargs["body"]["query"]shoulds = query["bool"]["filter"][0]["bool"]["should"]- fields = {next(iter(c["term"])): next(iter(c["term"].values())) for c in shoulds}- assert fields["connector_file_id"] == "file-id-1"- assert fields["document_id"] == "file-id-1"+ fields = {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}+ assert fields["connector_file_id"] == ["file-id-1"]+ assert fields["document_id"] == ["file-id-1"]++ # ...and stay scoped to this connector type, so an id that collides with a+ # different connector's id can't take its chunks down with it.+ terms = [f["term"] for f in query["bool"]["filter"] if "term" in f]+ assert {"connector_type": "sharepoint"} in terms@pytest.mark.asyncio@@ -321,6 +337,10 @@ def _wire_langflow_processor(return _make_search_response(connector_id_exists)if document_id:return _make_search_response(hash_exists)+ if isinstance(body, dict) and "aggs" in body:+ # Bulk duplicate check (terms aggregation over filename aliases).+ return _make_filename_agg_response(filename_exists)+ # Hits-based filename scroll (delete_document_by_filename id collection).return _make_search_response(filename_exists)opensearch_client.search = mock_search@@ -581,10 +601,14 @@ async def test_rename_cleanup_matches_both_id_fields(monkeypatch, backend_write_)shoulds = captured["query"]["bool"]["filter"][0]["bool"]["should"]- fields = {next(iter(c["term"])) for c in shoulds}+ fields = {next(iter(c["terms"])) for c in shoulds}assert fields == {"document_id", "connector_file_id", "connector_file_id.keyword"}excluded = captured["query"]["bool"]["must_not"][0]["terms"]["filename"]assert set(get_filename_aliases("Renamed.pdf")).issubset(set(excluded))+ # Scoped to this connector type, matching the value the chunks were indexed+ # under, so a colliding id from another connector is left alone.+ terms = [f["term"] for f in captured["query"]["bool"]["filter"] if "term" in f]+ assert {"connector_type": "sharepoint"} in terms@pytest.mark.asynciodiff --git a/tests/unit/test_connector_sync_bucket_filter_shared.py b/tests/unit/test_connector_sync_bucket_filter_shared.pyindex 5307edbed..cc622e63b 100644--- a/tests/unit/test_connector_sync_bucket_filter_shared.py+++ b/tests/unit/test_connector_sync_bucket_filter_shared.py@@ -54,6 +54,10 @@ async def test_bucket_filter_sync_forwards_shared_to_new_and_changed_batches():user.user_id = "user-1"with (+ patch(+ "api.connectors.has_effective_permission",+ new=AsyncMock(return_value=True),+ ),patch("api.connectors.get_synced_file_ids_for_connector",new=AsyncMock(return_value=(["changed-file-1"], [], "connector_file_id")),diff --git a/tests/unit/test_opensearch_filenames.py b/tests/unit/test_opensearch_filenames.pynew file mode 100644index 000000000..b360a80fc--- /dev/null+++ b/tests/unit/test_opensearch_filenames.py@@ -0,0 +1,74 @@+"""Shared filename-existence executor (utils/opensearch_filenames).++All duplicate-detection altitudes (UI pre-check endpoint, connector duplicate+classifier, processor backstop) go through these two functions, so their+query semantics cannot drift apart.+"""++from unittest.mock import AsyncMock++import pytest++from utils.opensearch_filenames import filename_exists, find_existing_filenames+++def _agg_response(*filenames: str) -> dict:+ return {+ "aggregations": {"filenames": {"buckets": [{"key": f, "doc_count": 1} for f in filenames]}}+ }+++async def test_find_existing_filenames_uses_terms_aggregation():+ client = AsyncMock()+ client.search.return_value = _agg_response("a.pdf")++ result = await find_existing_filenames(["a.pdf", "b.pdf"], client, "docs")++ assert result == {"a.pdf"}+ body = client.search.await_args.kwargs["body"]+ # Aggregation, not hits: exact results regardless of per-file chunk counts.+ assert body["size"] == 0+ assert set(body["query"]["terms"]["filename"]) == {"a.pdf", "b.pdf"}+ assert "aggs" in body+++async def test_find_existing_filenames_dedupes_and_drops_empty():+ client = AsyncMock()+ client.search.return_value = _agg_response()++ await find_existing_filenames(["a.pdf", "a.pdf", "", None], client, "docs")++ assert client.search.await_args.kwargs["body"]["query"]["terms"]["filename"] == ["a.pdf"]+++async def test_find_existing_filenames_empty_input_short_circuits():+ client = AsyncMock()+ assert await find_existing_filenames([], client, "docs") == set()+ client.search.assert_not_awaited()+++async def test_filename_exists_expands_aliases():+ client = AsyncMock()+ client.search.return_value = _agg_response("report.md")++ assert await filename_exists("report.txt", client, "docs") is True+ queried = set(client.search.await_args.kwargs["body"]["query"]["terms"]["filename"])+ assert {"report.txt", "report.md"}.issubset(queried)+++async def test_filename_exists_false_when_no_alias_indexed():+ client = AsyncMock()+ client.search.return_value = _agg_response()+ assert await filename_exists("report.pdf", client, "docs") is False+++async def test_filename_exists_blank_input_short_circuits():+ client = AsyncMock()+ assert await filename_exists("", client, "docs") is False+ client.search.assert_not_awaited()diff --git a/tests/unit/test_processor_refresh_uses_admin_client.py b/tests/unit/test_processor_refresh_uses_admin_client.pyindex ad1be9cec..bfdcf21f6 100644--- a/tests/unit/test_processor_refresh_uses_admin_client.py+++ b/tests/unit/test_processor_refresh_uses_admin_client.py@@ -54,7 +54,7 @@ async def test_document_processor_refresh_uses_admin_client(monkeypatch, tmp_patsession_manager=session_manager,)processor.check_filename_exists = AsyncMock(return_value=True)- processor.delete_document_by_filename = AsyncMock()+ processor.delete_document_by_filename = AsyncMock(return_value=1)processor.process_document_standard = AsyncMock(return_value={"status": "indexed", "id": "hash-1"})@@ -98,7 +98,7 @@ async def test_langflow_processor_refresh_uses_admin_client(monkeypatch, tmp_patreplace_duplicates=True,)processor.check_filename_exists = AsyncMock(return_value=True)- processor.delete_document_by_filename = AsyncMock()+ processor.delete_document_by_filename = AsyncMock(return_value=1)# The Langflow path reads the file off disk after the refresh.item = tmp_path / "report.pdf"diff --git a/tests/unit/test_processors_filename_alias_retry.py b/tests/unit/test_processors_filename_alias_retry.pyindex 7a690a4ef..45e0d7c55 100644--- a/tests/unit/test_processors_filename_alias_retry.py+++ b/tests/unit/test_processors_filename_alias_retry.py@@ -1,4 +1,6 @@-import asyncio+"""check_filename_exists issues ONE bulk aliases query (shared semantics) and+retries it on transient failures."""+from unittest.mock import AsyncMockimport pytest@@ -7,31 +9,40 @@ from models.processors import TaskProcessor@pytest.mark.asyncio-async def test_check_filename_exists_does_not_requery_successfully_checked_aliases_on_retry():- """Once an alias is checked successfully, retries should continue from pending aliases."""+async def test_check_filename_exists_bulk_queries_all_aliases_and_retries():+ """All filename aliases go into a single terms-aggregation query; a+ transient failure retries the same bulk query."""processor = TaskProcessor()opensearch_client = AsyncMock()+ calls = []+async def _search_side_effect(*, index, body):- candidate = body["query"]["term"]["filename"]- if candidate == "report.md" and _search_side_effect.md_calls == 0:- _search_side_effect.md_calls += 1- raise asyncio.TimeoutError("transient timeout")- return {"hits": {"hits": []}}+ calls.append(body)+ if len(calls) == 1:+ raise TimeoutError("transient timeout")+ return {"aggregations": {"filenames": {"buckets": []}}}- _search_side_effect.md_calls = 0opensearch_client.search.side_effect = _search_side_effectexists = await processor.check_filename_exists("report.txt", opensearch_client)assert exists is False+ assert len(calls) == 2 # first attempt timed out, retry succeeded+ assert calls[0] == calls[1] # retry sends the identical query+ for body in calls:+ queried = set(body["query"]["terms"]["filename"])+ # Both the .txt name and its .md ingestion alias are covered in one query.+ assert {"report.txt", "report.md"}.issubset(queried)+++async def test_check_filename_exists_true_when_any_alias_has_chunks():+ processor = TaskProcessor()+ opensearch_client = AsyncMock()+ opensearch_client.search.return_value = {+ "aggregations": {"filenames": {"buckets": [{"key": "report.md", "doc_count": 3}]}}+ }- queried_candidates = [- call.kwargs["body"]["query"]["term"]["filename"]- for call in opensearch_client.search.await_args_list- ]- # Expected sequence:- # 1) report.txt succeeds (no hits)- # 2) report.md times out- # 3) retry continues from pending alias only -> report.md- assert queried_candidates == ["report.txt", "report.md", "report.md"]+ assert await processor.check_filename_exists("report.txt", opensearch_client) is True+ opensearch_client.search.assert_awaited_once()diff --git a/tests/unit/test_shared_flag.py b/tests/unit/test_shared_flag.pyindex a6c541de6..407c25894 100644--- a/tests/unit/test_shared_flag.py+++ b/tests/unit/test_shared_flag.py@@ -213,12 +213,15 @@ async def test_non_cos_connector_rejects_shared_true():@pytest.mark.asyncio-async def test_ibm_cos_shared_true_does_not_hit_guard():+async def test_ibm_cos_shared_true_does_not_hit_guard(monkeypatch):"""shared=True with ibm_cos should NOT be rejected by the guard."""from unittest.mock import AsyncMock, MagicMock+ from api import connectors as connectors_apifrom api.connectors import ConnectorSyncBody, connector_sync+ monkeypatch.setattr(connectors_api, "has_effective_permission", AsyncMock(return_value=True))+body = ConnectorSyncBody(shared=True)connector_service = MagicMock()session_manager = MagicMock()@@ -241,3 +244,32 @@ async def test_ibm_cos_shared_true_does_not_hit_guard():)# 404 = "no active connections" error, not 400 = guard rejectionassert response.status_code == 404+++async def test_ibm_cos_shared_sync_denied_without_delete_anonymous_permission(monkeypatch):+ """shared=True requires knowledge:delete:anonymous for all sync paths."""+ from unittest.mock import AsyncMock, MagicMock++ from api import connectors as connectors_api+ from api.connectors import ConnectorSyncBody, connector_sync++ monkeypatch.setattr(connectors_api, "has_effective_permission", AsyncMock(return_value=False))+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())++ body = ConnectorSyncBody(shared=True)++ response = await connector_sync(+ connector_type="ibm_cos",+ body=body,+ request=MagicMock(),+ connector_service=MagicMock(),+ session_manager=MagicMock(),+ user=MagicMock(jwt_token="token", user_id="user-1"),+ session=AsyncMock(),+ rbac=MagicMock(),+ )+ import json++ assert response.status_code == 403+ assert "knowledge:delete:anonymous" in json.loads(response.body)["error"]diff --git a/tests/unit/test_traditional_duplicate_handling.py b/tests/unit/test_traditional_duplicate_handling.pyindex 30570f5e9..ea6f6ef27 100644--- a/tests/unit/test_traditional_duplicate_handling.py+++ b/tests/unit/test_traditional_duplicate_handling.py@@ -1,16 +1,42 @@-"""Unit tests for duplicate filename handling in DocumentFileProcessor."""+"""Unit tests for the unified duplicate-filename gate across processors.-from unittest.mock import ANY, AsyncMock, MagicMock, patch+All processors resolve a filename duplicate the same way via+``TaskProcessor.resolve_duplicate_filename``: with ``replace_duplicates=False``+the file is SKIPPED (counted successful, with a warning); with+``replace_duplicates=True`` the existing document is deleted and ingestion+proceeds.+"""++from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from models.processors import DocumentFileProcessor+from models.processors import (+ DUPLICATE_FILENAME_WARNING,+ DocumentFileProcessor,+ LangflowFileProcessor,+ S3FileProcessor,+)from models.tasks import FileTask, TaskStatus, UploadTask+DUPLICATE_SKIPPED_RESULT = {+ "status": "skipped",+ "reason": "duplicate_filename",+ "warning": DUPLICATE_FILENAME_WARNING,+}+++def _assert_duplicate_skipped(upload_task: UploadTask, file_task: FileTask) -> None:+ assert file_task.status == TaskStatus.SKIPPED+ assert file_task.error is None+ assert file_task.result == DUPLICATE_SKIPPED_RESULT+ assert upload_task.failed_files == 0+ assert upload_task.successful_files == 1+@pytest.mark.asyncioasync def test_traditional_processor_duplicate_exists_no_replace():- """Verify that if a duplicate file exists and replace_duplicates is False, processing fails."""+ """A duplicate with replace_duplicates=False is skipped, not failed."""mock_doc_service = MagicMock()mock_models_service = MagicMock()mock_session_manager = MagicMock()@@ -36,10 +62,7 @@ async def test_traditional_processor_duplicate_exists_no_replace():await processor.process_item(upload_task, "/tmp/test.txt", file_task)- assert file_task.status == TaskStatus.FAILED- assert "already exists" in file_task.error- assert upload_task.failed_files == 1- assert upload_task.successful_files == 0+ _assert_duplicate_skipped(upload_task, file_task)processor.check_filename_exists.assert_called_once()processor.delete_document_by_filename.assert_not_called()@@ -69,7 +92,7 @@ async def test_traditional_processor_duplicate_exists_with_replace():# Mock base class methods directly on the instance to ensure perfect isolationprocessor.check_filename_exists = AsyncMock(return_value=True)- processor.delete_document_by_filename = AsyncMock()+ processor.delete_document_by_filename = AsyncMock(return_value=1)processor.process_document_standard = AsyncMock(return_value={"status": "indexed"})upload_task = UploadTask(task_id="task-123", total_files=1)@@ -87,12 +110,179 @@ async def test_traditional_processor_duplicate_exists_with_replace():assert upload_task.successful_files == 1processor.check_filename_exists.assert_called_once()- processor.delete_document_by_filename.assert_awaited_once_with(- "test.txt",- ANY,- owner_user_id="user-123",- )+ processor.delete_document_by_filename.assert_called_once()+ # The delete must be owner-scoped so it actually removes the old chunks.+ delete_call = processor.delete_document_by_filename.call_args+ assert delete_call.kwargs["owner_user_id"] == "user-123"processor.process_document_standard.assert_called_once()mock_session_manager.get_user_opensearch_client.assert_called_once_with("user-123", "mock-token")+++async def test_langflow_processor_duplicate_exists_no_replace():+ """LangflowFileProcessor skips duplicates with the same uniform payload."""+ mock_session_manager = MagicMock()+ langflow_file_service = MagicMock()+ langflow_file_service.upload_and_ingest_file = AsyncMock()++ processor = LangflowFileProcessor(+ langflow_file_service=langflow_file_service,+ session_manager=mock_session_manager,+ owner_user_id="user-123",+ jwt_token="mock-token",+ replace_duplicates=False,+ )+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock()++ upload_task = UploadTask(task_id="task-123", total_files=1)+ file_task = FileTask(file_path="/tmp/test.txt", filename="test.txt")++ await processor.process_item(upload_task, "/tmp/test.txt", file_task)++ _assert_duplicate_skipped(upload_task, file_task)+ processor.delete_document_by_filename.assert_not_called()+ langflow_file_service.upload_and_ingest_file.assert_not_called()+++async def test_resolve_duplicate_skips_when_delete_removes_nothing():+ """If replace is requested but deletion removes 0 chunks (e.g. owner_user_id+ missing on a private sync), resolve_duplicate_filename must return 'skip'+ instead of falsely reporting 'replaced'."""+ from models.processors import TaskProcessor++ processor = TaskProcessor()+ opensearch_client = AsyncMock()++ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock(return_value=0)++ result = await processor.resolve_duplicate_filename(+ "report.pdf",+ opensearch_client,+ replace=True,+ owner_user_id=None,+ )++ assert result == "skip"+ processor.delete_document_by_filename.assert_awaited_once()+++async def test_delete_document_by_filename_shared_without_owner(monkeypatch):+ """shared=True with no owner_user_id must use the anonymous filename query+ instead of silently skipping deletion."""+ from types import SimpleNamespace++ from models.processors import TaskProcessor++ admin_client = AsyncMock()+ admin_client.delete = AsyncMock(return_value={"result": "deleted"})+ monkeypatch.setattr(+ "config.settings.clients",+ SimpleNamespace(opensearch=admin_client),+ )+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}+ )++ processor = TaskProcessor()+ deleted = await processor.delete_document_by_filename(+ "report.pdf",+ opensearch_client,+ owner_user_id=None,+ shared=True,+ )++ assert deleted == 1+ search_body = opensearch_client.search.await_args.kwargs["body"]+ filters = search_body["query"]["bool"]["filter"]+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters+++def _build_s3_processor(replace_duplicates: bool) -> S3FileProcessor:+ document_service = MagicMock()+ document_service.session_manager = MagicMock()+ processor = S3FileProcessor(+ document_service,+ bucket="test-bucket",+ s3_client=MagicMock(),+ owner_user_id="user-123",+ jwt_token="mock-token",+ models_service=MagicMock(),+ docling_service=MagicMock(),+ replace_duplicates=replace_duplicates,+ )+ return processor+++async def test_s3_processor_duplicate_exists_no_replace():+ """S3FileProcessor now has the same duplicate gate: same-key duplicates+ are skipped before the object is even downloaded."""+ processor = _build_s3_processor(replace_duplicates=False)+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock()++ upload_task = UploadTask(task_id="task-123", total_files=1)+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")++ await processor.process_item(upload_task, "docs/report.pdf", file_task)++ _assert_duplicate_skipped(upload_task, file_task)+ processor.delete_document_by_filename.assert_not_called()+ processor.s3_client.download_fileobj.assert_not_called()+++async def test_s3_processor_duplicate_exists_with_replace(tmp_path):+ """With replace_duplicates=True the old document is deleted and the object+ is downloaded and processed."""+ processor = _build_s3_processor(replace_duplicates=True)+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock(return_value=1)+ processor.process_document_standard = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )+ processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})++ upload_task = UploadTask(task_id="task-123", total_files=1)+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")++ with patch("models.processors.hash_id", return_value="dummy-hash"):+ await processor.process_item(upload_task, "docs/report.pdf", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ assert upload_task.successful_files == 1+ processor.delete_document_by_filename.assert_called_once()+ assert processor.delete_document_by_filename.call_args.kwargs["owner_user_id"] == "user-123"+ processor.s3_client.download_fileobj.assert_called_once()+ processor.process_document_standard.assert_awaited_once()+++async def test_s3_processor_no_duplicate_proceeds():+ """No duplicate: the gate is a no-op and processing continues."""+ processor = _build_s3_processor(replace_duplicates=False)+ processor.check_filename_exists = AsyncMock(return_value=False)+ processor.delete_document_by_filename = AsyncMock()+ processor.process_document_standard = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )+ processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})++ upload_task = UploadTask(task_id="task-123", total_files=1)+ file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")++ with patch("models.processors.hash_id", return_value="dummy-hash"):+ await processor.process_item(upload_task, "docs/report.pdf", file_task)++ assert file_task.status == TaskStatus.COMPLETED+ processor.delete_document_by_filename.assert_not_called()+ processor.process_document_standard.assert_awaited_once()