OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: detect cloud-connector duplicates ingested from other sources (#2404)
Rico Furtado9aa4aceba48f
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex 2d5acd5a3..9f554ffe7 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -98,7 +98,6 @@ export function SharedBucketView({const [pendingDuplicates, setPendingDuplicates] = useState<{duplicateNames: string[];duplicateCount: number;- duplicateFiles: BucketDuplicateFile[];nonDuplicateFiles: BucketDuplicateFile[];} | null>(null);const isOverwriteConfirmedRef = useRef(false);@@ -177,9 +176,12 @@ export function SharedBucketView({};// Full bucket sync: backend classifies new/changed/unchanged itself and- // re-ingests both new and changed blobs (used both when there are no- // duplicates and when the user chooses to overwrite them).- const runBucketSync = () => {+ // re-ingests both new and changed blobs. With `replaceDuplicates`, it instead+ // re-ingests the whole selection and replaces what is already indexed — the+ // overwrite path. Either way the bucket names travel instead of the file+ // list, so the request stays the same size for a 10-object bucket and a+ // 10,000-object one.+ const runBucketSync = (replaceDuplicates = false) => {syncMutation.mutate({connectorType: connector.type,@@ -188,6 +190,7 @@ export function SharedBucketView({selected_files: [],bucket_filter: Array.from(selectedBuckets),settings: ingestSettings,+ replace_duplicates: replaceDuplicates,shared: showSharedToggle? (ingestSettings.shared ?? false): undefined,@@ -197,15 +200,11 @@ export function SharedBucketView({);};- // Sync explicit files. `replaceDuplicates` forces an unconditional- // re-ingest (bypassing the bucket_filter path's modified-time gate) —- // used when the user confirms "Overwrite duplicates" so an already- // up-to-date file still gets re-ingested, matching what "overwrite" means- // for the OAuth connectors.- const runSelectedFilesSync = (- files: BucketDuplicateFile[],- replaceDuplicates = false,- ) => {+ // Sync an explicit subset of the listing — how "skip duplicates" ingests+ // only the files that are not already indexed. (Overwrite goes through+ // runBucketSync(true) instead: it needs the whole selection, and naming the+ // buckets says that in one line.)+ const runSelectedFilesSync = (files: BucketDuplicateFile[]) => {syncMutation.mutate({connectorType: connector.type,@@ -213,7 +212,7 @@ export function SharedBucketView({connection_id: connector.connectionId!,selected_files: files,settings: ingestSettings,- replace_duplicates: replaceDuplicates,+ replace_duplicates: false,shared: showSharedToggle? (ingestSettings.shared ?? false): undefined,@@ -272,7 +271,6 @@ export function SharedBucketView({setPendingDuplicates({duplicateNames,duplicateCount,- duplicateFiles: checkData.duplicate_files || [],nonDuplicateFiles: checkData.non_duplicate_files || [],});setDuplicateDialogOpen(true);@@ -289,8 +287,7 @@ export function SharedBucketView({const handleOverwriteDuplicates = () => {if (!pendingDuplicates) return;isOverwriteConfirmedRef.current = true;- const { duplicateFiles, nonDuplicateFiles } = pendingDuplicates;- runSelectedFilesSync([...duplicateFiles, ...nonDuplicateFiles], true);+ runBucketSync(true);setPendingDuplicates(null);};diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 9afe02f56..9330de8ae 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -940,6 +940,7 @@ async def _sync_existing_connector_files(ingest_settings=ingest_settings,replace_duplicates=True,shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)else:if max_files is not None:@@ -952,6 +953,7 @@ async def _sync_existing_connector_files(ingest_settings=ingest_settings,replace_duplicates=_connector_sync_should_replace(connector_type),shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)else:# Fallback: use filename filtering (for Langflow-ingested files without document_id)@@ -969,11 +971,18 @@ async def _sync_existing_connector_files(ingest_settings=ingest_settings,replace_duplicates=_connector_sync_should_replace(connector_type),shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)return {"outcome": "synced", "task_id": task_id}class ConnectorSyncBody(BaseModel):+ # The connection this sync targets. The upload UI is rendered per connection+ # and sends it on every request; connector_check_duplicates resolves the same+ # id, so honouring it here keeps the dialog and the sync it confirms pointed+ # at one connection. Unset (or unknown) falls back to the first that+ # authenticates, as before.+ connection_id: str | None = Nonemax_files: int | None = Noneselected_files: list[Any] | None = None# When True, ingest ALL files from the connector (bypasses the existing-files gate).@@ -985,7 +994,9 @@ class ConnectorSyncBody(BaseModel):settings: dict[str, Any] | None = None# When True, files whose filename already exists in the index are replaced# rather than failing. Set by the provider upload UI after the user confirms- # overwrite in the duplicate dialog.+ # overwrite in the duplicate dialog. Honored for an explicit ``selected_files``+ # sync and for a whole-container ``bucket_filter`` sync, where it also+ # bypasses the modified-time/etag gate (see connector_sync).replace_duplicates: bool = False# When True (OSS only for now; SaaS deferred), run the ingest in preview mode# (same as direct upload). Honored only when is_ingest_preview_enabled().@@ -1141,6 +1152,161 @@ def _connector_scoped_to_buckets(connector, bucket_names: list[str]):return scoped+def _cleaned_blob_filename(file_info: dict[str, Any]) -> str:+ """The name a remote blob would be indexed under, cleaned exactly as+ ingestion cleans it — so name comparisons here mean what they say."""+ from utils.file_utils import clean_connector_filename++ return clean_connector_filename(+ file_info.get("name", ""),+ file_info.get("mimeType") or file_info.get("mime_type") or file_info.get("mimetype") or "",+ )+++# Ids travel to OpenSearch in `terms` clauses, which are capped by+# index.max_terms_count (65_536 by default). Whole-container syncs can exceed+# that, so id lookups go out in batches.+_ID_LOOKUP_BATCH_SIZE = 1024+++async def _bucket_blob_ids_already_synced(+ file_ids: list[str],+ connector_type: str,+ session_manager,+ user_id: str,+ jwt_token: str | None,+) -> set[str]:+ """Which of exactly these blob ids already have chunks under this connector.++ ``get_synced_file_ids_for_connector`` answers the same question for the+ whole connector in one terms aggregation, which is capped at+ ``OPENSEARCH_TERMS_AGG_LIMIT``: past the cap it omits ids that ARE ingested.+ Reading "absent from that set" as "this connector never synced it" would+ then turn an already-synced blob into a name collision — its own indexed+ document is what holds the name — and re-ingest it under overwrite, in+ exactly the large containers where the cap is reached. Asking about the+ handful of ids actually in question keeps the answer independent of how much+ the connector has ingested in total.++ The id is matched in both fields it can live in, as+ ``connectors.chunk_cleanup`` does: the standard ingest path stores it in+ ``connector_file_id``, the Langflow path in ``document_id``.+ """+ ids = [fid for fid in file_ids if fid]+ if not ids:+ return set()++ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)+ synced: set[str] = set()++ for start in range(0, len(ids), _ID_LOOKUP_BATCH_SIZE):+ batch = ids[start : start + _ID_LOOKUP_BATCH_SIZE]+ body: dict[str, Any] = {+ "size": 0,+ "query": {+ "bool": {+ "filter": [+ {"term": {"connector_type": connector_type}},+ {+ "bool": {+ "should": [+ {"terms": {"document_id": batch}},+ {"terms": {"connector_file_id": batch}},+ # Indices predating the explicit keyword+ # mapping dynamically mapped this as text.+ {"terms": {"connector_file_id.keyword": batch}},+ ],+ "minimum_should_match": 1,+ }+ },+ ]+ }+ },+ "aggs": {+ "connector_file_ids": {"terms": {"field": "connector_file_id", "size": len(batch)}},+ "document_ids": {"terms": {"field": "document_id", "size": len(batch)}},+ },+ }++ try:+ response = await opensearch_client.search(index=get_index_name(), body=body)+ except Exception as search_err:+ if _is_unmapped_keyword_agg_error(search_err):+ body["aggs"]["connector_file_ids"]["terms"]["field"] = "connector_file_id.keyword"+ response = await opensearch_client.search(index=get_index_name(), body=body)+ elif "index_not_found_exception" in str(search_err):+ return set()+ else:+ raise++ aggregations = response.get("aggregations", {})+ for agg_name in ("connector_file_ids", "document_ids"):+ for bucket in aggregations.get(agg_name, {}).get("buckets", []):+ if bucket.get("key"):+ synced.add(bucket["key"])++ # Only ids we asked about; the aggregations can also surface an id that+ # merely shares a chunk with one of them.+ return synced & set(ids)+++async def _bucket_blob_ids_with_indexed_filename(+ files: list[dict[str, Any]],+ session_manager,+ user_id: str,+ jwt_token: str | None,+) -> dict[str, str]:+ """Map each blob whose filename already has indexed chunks — from any source,+ a direct upload, another connector, or an earlier sync of this one — to the+ indexed name it lands on.++ The name, not just the id, because two blobs in one container can land on+ the same one: a bucket connector names a blob after its key's basename, so+ ``a/report.pdf`` and ``b/report.pdf`` are both ``report.pdf``. Callers that+ act on the collision need to know which of them compete.++ One batched terms aggregation for the whole listing, through the same+ ``find_existing_filenames`` the OAuth connector classifier and the per-file+ processor backstop use, so the duplicate question gets one answer at every+ altitude. A missing index means nothing is indexed yet, not a failure.+ """+ from utils.file_utils import get_filename_aliases+ from utils.opensearch_filenames import find_existing_filenames++ aliases_by_id: dict[str, list[str]] = {}+ all_candidates: set[str] = set()+ for f in files:+ fid = f.get("id")+ if not fid:+ continue+ aliases = get_filename_aliases(_cleaned_blob_filename(f))+ aliases_by_id[fid] = aliases+ all_candidates.update(aliases)++ if not all_candidates:+ return {}++ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)+ try:+ existing_filenames = await find_existing_filenames(+ all_candidates, opensearch_client, get_index_name()+ )+ except Exception as search_err:+ if "index_not_found_exception" not in str(search_err):+ raise+ return {}++ # Listing order, so a caller resolving a contested name gets a stable winner+ # (bucket listings are ordered by key).+ matched: dict[str, str] = {}+ for fid, aliases in aliases_by_id.items():+ for alias in aliases:+ if alias in existing_filenames:+ matched[fid] = alias+ break+ return matched++async def _classify_bucket_connector_duplicates(connector,connector_type: str,@@ -1149,13 +1315,14 @@ async def _classify_bucket_connector_duplicates(user_id: str,jwt_token: str | None,) -> dict[str, Any]:- """Preview a bucket_filter sync: classify remote blobs new/changed/unchanged- without ingesting anything, mirroring the reconciliation in connector_sync's- bucket_filter branch. "changed" blobs are reported as duplicates (they would- overwrite an already-indexed version); "unchanged" blobs are silently- dropped (the real sync would skip them too); "new" blobs are returned as- ``non_duplicate_files`` so the caller can sync just those when the user- chooses to skip duplicates.+ """Preview a bucket_filter sync: classify remote blobs as duplicate or new+ without ingesting anything, so the upload UI can confirm an overwrite.++ A blob is a duplicate when it is already indexed under THIS connector type+ (id match) or when its filename is already in the index from any other+ source — a direct upload, a different connector. Blobs that match neither+ are returned as ``non_duplicate_files`` so the caller can sync just those+ when the user chooses to skip duplicates."""scoped_connector = _connector_scoped_to_buckets(connector, bucket_filter)all_files: list[dict[str, Any]] = []@@ -1184,33 +1351,39 @@ async def _classify_bucket_connector_duplicates()existing_set = set(existing_ids)- # Existence-based, like the OAuth connector duplicate check: any blob- # already ingested under this connector_type is a "duplicate" regardless- # of whether the remote copy is newer. (The real bucket_filter sync uses- # modified_time to auto-skip unchanged blobs on ITS OWN — that's a- # separate, silent optimization; the confirm dialog here is about whether- # the user wants to touch an already-indexed file at all, same as it- # would for Google Drive/OneDrive/SharePoint.)+ # The id half answers "did this connector already ingest this blob?"; the+ # filename half answers "is this name already in the index at all?" — the+ # question a direct upload, or an ingest through a different connector,+ # makes relevant. Only the id half existed here, so a bucket whose blobs+ # collide by name with files ingested elsewhere reported zero duplicates,+ # skipped the confirm dialog, and then had every colliding file skipped at+ # ingest time with "A file with this name already exists".+ name_taken_ids = await _bucket_blob_ids_with_indexed_filename(+ all_files, session_manager, user_id, jwt_token+ )+duplicate_files: list[dict[str, Any]] = []duplicate_names: list[str] = []non_duplicate_files: list[dict[str, Any]] = []+ total = 0for f in all_files:fid = f.get("id")if not fid:continue- if fid in existing_set:- response_file = _connector_file_response(f)+ total += 1+ response_file = _connector_file_response(f, cleaned_name=_cleaned_blob_filename(f))+ if fid in existing_set or fid in name_taken_ids:duplicate_files.append(response_file)duplicate_names.append(response_file["name"])else:- non_duplicate_files.append(_connector_file_response(f))+ non_duplicate_files.append(response_file)return {"duplicate_names": list(dict.fromkeys(duplicate_names)),"duplicate_files": duplicate_files,"non_duplicate_files": non_duplicate_files,"duplicate_count": len(duplicate_files),- "total_files": len(all_files),+ "total_files": total,}@@ -1540,9 +1713,27 @@ async def connector_sync(status_code=404,)+ # Try the requested connection first, then the rest. Ordering rather than+ # selecting outright keeps today's behaviour when that connection cannot+ # authenticate: the sync falls through to another, or reports that none+ # work, instead of failing later inside the connector.+ candidate_connections = active_connections+ if body.connection_id:+ requested = [c for c in active_connections if c.connection_id == body.connection_id]+ if requested:+ candidate_connections = requested + [+ c for c in active_connections if c.connection_id != body.connection_id+ ]+ else:+ logger.warning(+ "Requested connection is not active for this connector type",+ connector_type=connector_type,+ connection_id=body.connection_id,+ )+# Find the first connection that actually worksworking_connection = None- for connection in active_connections:+ for connection in candidate_connections:logger.debug("Testing connection authentication",connection_id=connection.connection_id,@@ -1629,6 +1820,7 @@ async def connector_sync(replace_duplicates=body.replace_duplicates,preview_mode=preview_mode,shared=body.shared,+ allow_anonymous_delete=allow_anonymous_delete,)elif body.sync_all or body.bucket_filter:# Full ingest: discover and ingest all files (or files from specific buckets).@@ -1665,6 +1857,23 @@ async def connector_sync(status_code=200,)+ # Carry the listing's names into the task so its file rows read+ # "report.pdf" rather than the raw "<bucket>::<key>" id, which is+ # all the task has until each blob is fetched.+ infos_by_id = {+ f["id"]: _connector_file_response(f) for f in all_files if f.get("id")+ }+ if not infos_by_id:+ # Nothing addressable: every listed blob came back without an+ # id, so there is no sync to start.+ return JSONResponse(+ {+ "status": "no_files",+ "message": "No files found in the selected buckets.",+ },+ status_code=200,+ )+# Classify each remote blob as new / changed / unchanged.existing_ids, _, _ = await get_synced_file_ids_for_connector(connector_type=connector_type,@@ -1680,12 +1889,52 @@ async def connector_sync(jwt_token=jwt_token,)+ # "Overwrite duplicates", confirmed in the UI dialog, is about+ # blobs whose name is already indexed but which this connector+ # has never synced — a direct upload, another connector. Nothing+ # we store about them can say whether the source is newer, so+ # change detection cannot decide them and they are replaced+ # outright. It deliberately does NOT cover blobs this connector+ # already synced: those keep the modified-time/etag gate, so+ # confirming an overwrite re-ingests the colliding files rather+ # than every object in the container.+ colliding_ids: set[str] = set()+ if body.replace_duplicates:+ # One winner per contested name. Two blobs in the same+ # container can land on the same indexed filename (a bucket+ # connector names a blob after its key's basename, so+ # a/report.pdf and b/report.pdf are both "report.pdf"), and+ # files within a task are ingested concurrently — replacing+ # one document from two of them at once has no defined+ # winner. The first in listing order takes the name; the+ # rest fall through to the per-file backstop, which skips+ # them as duplicates exactly as it does today.+ claimed_names: set[str] = set()+ collisions = await _bucket_blob_ids_with_indexed_filename(+ all_files, session_manager, user.user_id, jwt_token+ )+ # "Synced here already?" is asked of these ids directly+ # rather than read off existing_set, whose aggregation is+ # capped: above the cap an ingested blob is missing from it,+ # and would look like a collision with its own document.+ synced_collisions = await _bucket_blob_ids_already_synced(+ list(collisions), connector_type, session_manager, user.user_id, jwt_token+ )+ for fid, indexed_name in collisions.items():+ if fid in synced_collisions or indexed_name in claimed_names:+ continue+ claimed_names.add(indexed_name)+ colliding_ids.add(fid)+new_ids: list[str] = []- changed_ids: list[str] = []+ replace_ids: list[str] = []for f in all_files:fid = f.get("id")if not fid:continue+ if fid in colliding_ids:+ replace_ids.append(fid)+ continuestatus = classify_remote_file_change(fid,f.get("modified_time"),@@ -1696,7 +1945,7 @@ async def connector_sync(if status == "new":new_ids.append(fid)elif status == "changed":- changed_ids.append(fid)+ replace_ids.append(fid)# "unchanged" → skip; already ingested and not newer at source.logger.info(@@ -1704,11 +1953,12 @@ async def connector_sync(connector_type=connector_type,total=len(all_files),new=len(new_ids),- changed=len(changed_ids),- skipped=len(all_files) - len(new_ids) - len(changed_ids),+ replaced=len(replace_ids),+ overwritten_duplicates=len(colliding_ids),+ skipped=len(all_files) - len(new_ids) - len(replace_ids),)- if not new_ids and not changed_ids:+ if not new_ids and not replace_ids:return JSONResponse({"status": "no_files",@@ -1717,10 +1967,10 @@ async def connector_sync(status_code=200,)- # Two batches: new files are created; changed files replace the- # indexed copy (replace_duplicates=True bypasses the filename-skip- # and deletes stale chunks before re-ingest). replace is batch-level,- # hence the split.+ # Two batches: new files are created; changed and overwritten+ # files replace the indexed copy (replace_duplicates=True bypasses+ # the filename-skip and deletes stale chunks before re-ingest).+ # replace is batch-level, hence the split.if new_ids:task_ids.append(await connector_service.sync_specific_files(@@ -1728,22 +1978,26 @@ async def connector_sync(user.user_id,new_ids,jwt_token=jwt_token,+ file_infos=[infos_by_id[fid] for fid in new_ids],ingest_settings=body.settings,preview_mode=preview_mode,shared=body.shared,+ allow_anonymous_delete=allow_anonymous_delete,))- if changed_ids:+ if replace_ids:task_ids.append(await connector_service.sync_specific_files(working_connection.connection_id,user.user_id,- changed_ids,+ replace_ids,jwt_token=jwt_token,+ file_infos=[infos_by_id[fid] for fid in replace_ids],ingest_settings=body.settings,replace_duplicates=True,preview_mode=preview_mode,shared=body.shared,+ allow_anonymous_delete=allow_anonymous_delete,))else:@@ -1755,6 +2009,7 @@ async def connector_sync(jwt_token=jwt_token,ingest_settings=body.settings,shared=body.shared,+ allow_anonymous_delete=allow_anonymous_delete,)else:# No files specified - sync only files already in OpenSearch for this connectordiff --git a/src/connectors/service.py b/src/connectors/service.pyindex c7bfa8de7..99ec51a3f 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -243,6 +243,7 @@ class ConnectorService:ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,shared: bool | None = False,+ allow_anonymous_delete: bool = True,) -> str:"""Sync files from a connector connection using existing task tracking system.@@ -352,6 +353,7 @@ class ConnectorService:replace_duplicates=replace_duplicates,connector_type=connector.CONNECTOR_TYPE,shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)# Use file IDs as items (no more fake file paths!)@@ -385,6 +387,7 @@ class ConnectorService:replace_duplicates: bool = False,preview_mode: bool = False,shared: bool | None = False,+ allow_anonymous_delete: bool = True,) -> str:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).@@ -526,6 +529,7 @@ class ConnectorService:replace_duplicates=replace_duplicates,connector_type=connector.CONNECTOR_TYPE,shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)# Create custom task using TaskServicediff --git a/src/models/processors.py b/src/models/processors.pyindex 77feb1a8f..bdee0021a 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -21,7 +21,7 @@ from utils.file_utils import ()from utils.hash_utils import hash_idfrom utils.logging_config import get_logger-from utils.opensearch_queries import build_replace_filename_query+from utils.opensearch_queries import build_owned_filename_query, build_replace_filename_queryfrom .tasks import FileTask, TaskStatus, UploadTask@@ -259,6 +259,7 @@ class TaskProcessor:replace: bool,owner_user_id: str | None,shared: bool = False,+ allow_anonymous_delete: bool = True,) -> Literal["proceed", "skip", "replaced"]:"""Single duplicate-filename policy shared by every processor.@@ -283,6 +284,7 @@ class TaskProcessor:opensearch_client,owner_user_id=owner_user_id,shared=shared,+ allow_anonymous_delete=allow_anonymous_delete,)if deleted == 0:logger.warning(@@ -322,22 +324,36 @@ class TaskProcessor:opensearch_client,owner_user_id: str | None = None,shared: bool = False,+ allow_anonymous_delete: bool = True,) -> int:"""Delete all chunks of a document with the given filename fromOpenSearch. Returns the number of chunks deleted.``shared`` describes how the replacement is about to be *written*, notwhat is already indexed, so it must not narrow what we delete: with an- owner in hand the scope is always "owned by this user OR ownerless".+ owner in hand the scope is "owned by this user OR ownerless".Choosing an owner-only scope for a shared document matched none of itschunks, and the caller read that zero as "nothing to replace" andskipped the file — leaving the stale copy in the index even though theduplicate check (which is owner-agnostic) had just found it and the user- had confirmed the overwrite."""+ had confirmed the overwrite.++ ``allow_anonymous_delete`` is the caller's resolved+ ``knowledge:delete:anonymous``. Ownerless chunks are visible to everyone+ in the instance, so replacing a document that turns out to be one is a+ deletion of shared content: without that permission the scope stays+ owner-only and someone else's shared document is left alone (the file+ then resolves as a duplicate the user may not replace). It defaults to+ True because callers that have not resolved the permission — uploads,+ the Langflow path, sample docs — keep their existing behaviour; see the+ note in the PR about closing that across every entry point.+ Deliberately ignored when ``shared`` is True: a shared write already+ required the permission upstream."""from config.settings import clients, get_index_namefrom utils.opensearch_delete import collect_visible_document_ids, delete_document_idsfrom utils.opensearch_queries import (build_anonymous_filename_query,+ build_owned_filename_query,build_replace_filename_query,)@@ -358,8 +374,10 @@ class TaskProcessor:)return 0- else:+ elif shared or allow_anonymous_delete:build_query = build_replace_filename_query+ else:+ build_query = build_owned_filename_querycandidate_filenames = get_filename_aliases(filename)if not candidate_filenames:@@ -900,6 +918,7 @@ class ConnectorFileProcessor(TaskProcessor):connector_type: str | None = None,preview_mode: bool = False,shared: bool | None = False,+ allow_anonymous_delete: bool = True,):super().__init__(document_service=document_service,@@ -918,6 +937,10 @@ class ConnectorFileProcessor(TaskProcessor):self.connector_type = connector_typeself.preview_mode = preview_modeself.shared = shared+ # The syncing user's resolved knowledge:delete:anonymous, threaded down+ # to the replace path so an overwrite cannot delete a shared document+ # the user is not allowed to delete.+ self.allow_anonymous_delete = allow_anonymous_deleteasync def _indexed_shared_state(self,@@ -995,9 +1018,20 @@ class ConnectorFileProcessor(TaskProcessor):since then. Without this, those chunks would keep whatever owner theygot on their original ingest forever, since a byte-identical re-syncnever reaches resolve_shared_owner_fields(). Scoped to chunks owned by- this user or already ownerless (matching the same boundary- delete_document_by_filename uses), so it can't touch another user's- private document that happens to share this filename.+ this user — and to ownerless ones only under the same+ ``knowledge:delete:anonymous`` boundary delete_document_by_filename+ uses — so it can't touch another user's document that happens to share+ this filename.++ That boundary matters here even more than it does for a delete. An+ ownerless document is visible to the whole instance, and this script+ writes an owner onto what it matches: without the permission check, a+ private sync of a file whose name collides with a shared document would+ quietly claim that document for the syncing user, taking it out of+ everyone else's view. ``_indexed_shared_state`` cannot prevent it — it+ keys on the connector file id, and a collision by definition comes from+ a document this connector never ingested, so the lookup returns None and+ ``shared`` falls back to the sync's own (private) intent.`shared` comes from _resolve_shared, never straight from self.shared: ona re-sync it reflects the file's current indexed state, so this is a@@ -1009,12 +1043,19 @@ class ConnectorFileProcessor(TaskProcessor):owner, owner_name, owner_email = resolve_shared_owner_fields(self.user_id, self.owner_name, self.owner_email, shared)+ # A shared write already cleared the permission upstream (connector_sync+ # rejects shared syncs without it), so it keeps the wider scope.+ build_query = (+ build_replace_filename_query+ if shared or self.allow_anonymous_delete+ else build_owned_filename_query+ )for candidate in get_filename_aliases(filename):try:await write_client.update_by_query(index=get_index_name(),body={- "query": build_replace_filename_query(candidate, self.user_id),+ "query": build_query(candidate, self.user_id),"script": {"source": """if (params.shared) {@@ -1185,6 +1226,7 @@ class ConnectorFileProcessor(TaskProcessor):replace=self.replace_duplicates,owner_user_id=self.user_id,shared=shared,+ allow_anonymous_delete=self.allow_anonymous_delete,)if duplicate_action == "skip":await self._reconcile_shared_owner(file_task.filename, shared)diff --git a/src/utils/opensearch_filenames.py b/src/utils/opensearch_filenames.pyindex 4f3b154a2..cfc1ad8f3 100644--- a/src/utils/opensearch_filenames.py+++ b/src/utils/opensearch_filenames.py@@ -16,6 +16,12 @@ from collections.abc import Iterablefrom utils.file_utils import get_filename_aliasesfrom utils.opensearch_queries import build_existing_filenames_agg_body+# OpenSearch rejects a terms query carrying more than `index.max_terms_count`+# values (default 65_536). Whole-container duplicate checks feed this function+# one candidate per remote blob — times the alias expansion — so a large bucket+# can exceed that on its own. Candidates go out in batches instead.+FILENAME_LOOKUP_BATCH_SIZE = 1024+async def find_existing_filenames(candidates: Iterable[str],@@ -31,12 +37,16 @@ async def find_existing_filenames(unique = sorted({c for c in candidates if c})if not unique:return set()- response = await opensearch_client.search(- index=index,- body=build_existing_filenames_agg_body(unique),- )- buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])- return {bucket["key"] for bucket in buckets if bucket.get("key")}+ found: set[str] = set()+ for start in range(0, len(unique), FILENAME_LOOKUP_BATCH_SIZE):+ batch = unique[start : start + FILENAME_LOOKUP_BATCH_SIZE]+ response = await opensearch_client.search(+ index=index,+ body=build_existing_filenames_agg_body(batch),+ )+ buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])+ found.update(bucket["key"] for bucket in buckets if bucket.get("key"))+ return foundasync def filename_exists(diff --git a/tests/unit/api/test_connector_bucket_duplicates.py b/tests/unit/api/test_connector_bucket_duplicates.pynew file mode 100644index 000000000..8ece9c4d9--- /dev/null+++ b/tests/unit/api/test_connector_bucket_duplicates.py@@ -0,0 +1,743 @@+"""Duplicate handling for whole-container (bucket_filter) connector ingests.++The bug these cover: the pre-check that decides whether the overwrite dialog+appears was id-based only — "did THIS connector type already ingest this+blob?" — while the per-file backstop in ConnectorFileProcessor is filename-based+across every source. A bucket whose blobs share names with files ingested by+direct upload therefore reported zero duplicates, skipped the dialog, and had+every colliding file skipped at ingest time with "A file with this name already+exists". The two altitudes now ask the same question, and an overwrite+confirmed in the dialog reaches the bucket sync path.+"""++import json+import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _json(response):+ return json.loads(response.body.decode())+++def _rbac_allowing(*perms: str):+ """RBAC stub granting exactly `perms` (unit tests run with enforcement on)."""+ rbac = MagicMock()+ granted = set(perms)++ async def has_permission(user_id, perm, role_override=None):+ return perm in granted++ rbac.has_permission = AsyncMock(side_effect=has_permission)+ return rbac+++def _session_manager_finding(*indexed_filenames: str, synced_ids: tuple[str, ...] = ()):+ """Session manager whose OpenSearch client reports these filenames indexed,+ and these blob ids already synced under the connector.++ Serves both queries the sync issues for real — the filename terms+ aggregation behind find_existing_filenames, and the per-id lookup that asks+ which of those blobs this connector already ingested — so the real query+ paths run rather than a stubbed-out classifier.+ """+ indexed = set(indexed_filenames)+ synced = set(synced_ids)+ client = AsyncMock()++ async def search(*, index, body):+ terms = body["query"].get("terms")+ if terms is not None:+ asked = terms["filename"]+ return {+ "aggregations": {+ "filenames": {"buckets": [{"key": name} for name in asked if name in indexed]}+ }+ }++ # The per-id "already synced here?" lookup.+ should = body["query"]["bool"]["filter"][1]["bool"]["should"]+ asked_ids = should[1]["terms"]["connector_file_id"]+ return {+ "aggregations": {+ "connector_file_ids": {+ "buckets": [{"key": fid} for fid in asked_ids if fid in synced]+ },+ "document_ids": {"buckets": []},+ }+ }++ client.search = AsyncMock(side_effect=search)+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)+ return session_manager, client+++def _bucket_connector(remote_files):+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)+ connector.bucket_names = None+ connector.list_files = AsyncMock(return_value={"files": remote_files, "next_page_token": None})+ return connector+++def _bucket_sync_service(remote_files, task_id="task-x"):+ connection = SimpleNamespace(connection_id="conn-1", is_active=True)+ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ service.get_connector = AsyncMock(return_value=_bucket_connector(remote_files))+ service.sync_specific_files = AsyncMock(return_value=task_id)+ return service+++# ---------------------------------------------------------------------------+# _classify_bucket_connector_duplicates — what the confirm dialog is told+# ---------------------------------------------------------------------------+++async def test_filename_indexed_by_another_source_is_a_duplicate(monkeypatch):+ """The reported bug: sample.pdf was uploaded directly, so no COS id matches+ it, but its name is taken. Without this the dialog never opened."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ session_manager, _ = _session_manager_finding("sample.pdf")++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=_bucket_connector(+ [+ {"id": "b::sample.pdf", "name": "sample.pdf"},+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},+ ]+ ),+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="alice",+ jwt_token="token",+ )++ assert result["duplicate_count"] == 1+ assert result["duplicate_names"] == ["sample.pdf"]+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::fresh.pdf"]+ assert result["total_files"] == 2+++async def test_already_synced_id_is_still_a_duplicate(monkeypatch):+ """The id half of the check survives: a blob this connector ingested before+ counts even when its indexed chunks are invisible to the filename lookup."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["b::already.pdf"], [], "connector_file_id")),+ )+ session_manager, _ = _session_manager_finding()++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=_bucket_connector(+ [+ {"id": "b::already.pdf", "name": "already.pdf"},+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},+ ]+ ),+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="alice",+ jwt_token="token",+ )++ assert result["duplicate_names"] == ["already.pdf"]+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::fresh.pdf"]+++async def test_alias_match_counts_as_duplicate(monkeypatch):+ """Legacy Langflow ingest indexes notes.txt as notes.md — the alias the+ processor backstop also honors."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ session_manager, _ = _session_manager_finding("notes.md")++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=_bucket_connector([{"id": "b::notes.txt", "name": "notes.txt"}]),+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="alice",+ jwt_token="token",+ )++ assert result["duplicate_names"] == ["notes.txt"]+++async def test_nothing_indexed_reports_no_duplicates(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ session_manager, _ = _session_manager_finding()++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=_bucket_connector([{"id": "b::a.pdf", "name": "a.pdf"}]),+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="alice",+ jwt_token="token",+ )++ assert result["duplicate_count"] == 0+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::a.pdf"]+++async def test_missing_index_is_not_an_error(monkeypatch):+ """First ingest on a fresh instance: no documents index yet, nothing is a+ duplicate. Any other search failure still propagates."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ client = AsyncMock()+ client.search = AsyncMock(side_effect=Exception("index_not_found_exception: idx"))+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)++ result = await connectors_api._classify_bucket_connector_duplicates(+ connector=_bucket_connector([{"id": "b::a.pdf", "name": "a.pdf"}]),+ connector_type="ibm_cos",+ bucket_filter=["b"],+ session_manager=session_manager,+ user_id="alice",+ jwt_token="token",+ )++ assert result["duplicate_count"] == 0+++# ---------------------------------------------------------------------------+# connector_sync — what "overwrite" does to a whole-container selection+# ---------------------------------------------------------------------------+++async def test_bucket_filter_overwrite_replaces_only_the_colliding_files(monkeypatch):+ """Overwrite targets the blobs whose name is indexed but which this+ connector never synced. Nothing we store about them can say whether the+ source is newer, so change detection cannot decide them — they are replaced+ outright, while a blob this connector already synced and that has not+ changed is still skipped."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["b::synced.pdf"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_state_map",+ AsyncMock(+ return_value={+ "b::synced.pdf": connectors_api.SyncedFileState(+ modified_time_ms=1704067200000.0, content_etag=None+ )+ }+ ),+ )++ remote_files = [+ # Uploaded directly before this connector existed: name taken, id new.+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"},+ # Synced by this connector and unchanged at source.+ {"id": "b::synced.pdf", "name": "synced.pdf", "modified_time": "2024-01-01T00:00:00Z"},+ # Not indexed anywhere.+ {"id": "b::fresh.pdf", "name": "fresh.pdf", "modified_time": "2024-01-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)+ session_manager, _ = _session_manager_finding(+ "uploaded.pdf", "synced.pdf", synced_ids=("b::synced.pdf",)+ )++ response = await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ assert response.status_code == 201+ assert service.sync_specific_files.await_count == 2+ new_call, replace_call = service.sync_specific_files.await_args_list+ assert new_call.args[2] == ["b::fresh.pdf"]+ assert new_call.kwargs.get("replace_duplicates", False) is False+ # Only the name collision is overwritten; the unchanged synced blob is not+ # in either batch.+ assert replace_call.args[2] == ["b::uploaded.pdf"]+ assert replace_call.kwargs["replace_duplicates"] is True+++async def test_bucket_filter_overwrite_batches_collisions_with_changed_blobs(monkeypatch):+ """A blob changed at source and a blob colliding by name both need the same+ replace semantics, so they ride in one batch."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["b::changed.pdf"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_state_map",+ AsyncMock(+ return_value={+ "b::changed.pdf": connectors_api.SyncedFileState(+ modified_time_ms=1704067200000.0, content_etag=None+ )+ }+ ),+ )++ remote_files = [+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "b::changed.pdf", "name": "changed.pdf", "modified_time": "2024-06-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)+ session_manager, _ = _session_manager_finding(+ "uploaded.pdf", "changed.pdf", synced_ids=("b::changed.pdf",)+ )++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ assert service.sync_specific_files.await_count == 1+ call = service.sync_specific_files.await_args+ assert sorted(call.args[2]) == ["b::changed.pdf", "b::uploaded.pdf"]+ assert call.kwargs["replace_duplicates"] is True+++async def test_bucket_filter_without_overwrite_never_looks_up_filenames(monkeypatch):+ """Declining the overwrite leaves the sync exactly as it was: the colliding+ blob rides in the plain batch and the processor backstop skips it. No+ filename lookup is issued."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"}+ ]+ service = _bucket_sync_service(remote_files)+ session_manager, client = _session_manager_finding("uploaded.pdf")++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["b"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ client.search.assert_not_awaited()+ call = service.sync_specific_files.await_args+ assert call.args[2] == ["b::uploaded.pdf"]+ assert call.kwargs.get("replace_duplicates", False) is False+++async def test_bucket_filter_sync_carries_filenames_into_the_task(monkeypatch):+ """Without file_infos the task view shows the raw "<bucket>::<key>" id until+ each blob is fetched."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [+ {"id": "b::deep/report.pdf", "name": "report.pdf", "modified_time": None},+ ]+ service = _bucket_sync_service(remote_files)++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["b"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ file_infos = service.sync_specific_files.await_args.kwargs["file_infos"]+ assert [f["name"] for f in file_infos] == ["report.pdf"]+ assert [f["id"] for f in file_infos] == ["b::deep/report.pdf"]+++async def test_bucket_filter_listing_without_ids_starts_no_sync(monkeypatch):+ """Nothing addressable in the listing: report no_files rather than starting+ an empty task."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ service = _bucket_sync_service([{"name": "no-id.pdf"}])++ response = await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ assert response.status_code == 200+ assert _json(response)["status"] == "no_files"+ service.sync_specific_files.assert_not_awaited()+++async def test_overwrite_gives_one_winner_when_two_blobs_share_a_name(monkeypatch):+ """A bucket connector names a blob after its key's basename, so a/report.pdf+ and b/report.pdf both land on "report.pdf". Files within a task are ingested+ concurrently, so letting both replace the same document has no defined+ winner. The first in listing order takes the name; the other rides in the+ plain batch, where the per-file backstop skips it as a duplicate.+ """+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [+ {"id": "b::a/report.pdf", "name": "report.pdf", "modified_time": None},+ {"id": "b::c/report.pdf", "name": "report.pdf", "modified_time": None},+ ]+ service = _bucket_sync_service(remote_files)+ session_manager, _ = _session_manager_finding("report.pdf")++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ assert service.sync_specific_files.await_count == 2+ new_call, replace_call = service.sync_specific_files.await_args_list+ assert replace_call.args[2] == ["b::a/report.pdf"]+ assert replace_call.kwargs["replace_duplicates"] is True+ assert new_call.args[2] == ["b::c/report.pdf"]+ assert new_call.kwargs.get("replace_duplicates", False) is False+++async def test_alias_collision_between_two_blobs_gets_one_winner(monkeypatch):+ """notes.txt and notes.md are ingestion aliases of one another, so two blobs+ can contest a name without sharing one."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [+ {"id": "b::notes.txt", "name": "notes.txt", "modified_time": None},+ {"id": "b::notes.md", "name": "notes.md", "modified_time": None},+ ]+ service = _bucket_sync_service(remote_files)+ session_manager, _ = _session_manager_finding("notes.md")++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ replace_calls = [+ c for c in service.sync_specific_files.await_args_list if c.kwargs.get("replace_duplicates")+ ]+ assert len(replace_calls) == 1+ assert replace_calls[0].args[2] == ["b::notes.txt"]++[email protected]("granted", [True, False])+async def test_overwrite_carries_the_users_anonymous_delete_permission(monkeypatch, granted):+ """Replacing a duplicate can delete a shared (ownerless) document, which is+ what knowledge:delete:anonymous governs. connector_sync already resolves it+ for orphan cleanup; the sync must hand the same answer to the replace path+ rather than letting it widen on its own."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [{"id": "b::report.pdf", "name": "report.pdf", "modified_time": None}]+ service = _bucket_sync_service(remote_files)+ session_manager, _ = _session_manager_finding("report.pdf")++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous") if granted else _rbac_allowing(),+ )++ call = service.sync_specific_files.await_args+ assert call.kwargs["replace_duplicates"] is True+ assert call.kwargs["allow_anonymous_delete"] is granted+++async def test_overwrite_ignores_a_truncated_synced_id_listing(monkeypatch):+ """get_synced_file_ids_for_connector's terms aggregation is capped at+ OPENSEARCH_TERMS_AGG_LIMIT, so a container past the cap reports only some of+ the ids it has ingested. An omitted blob must not become a collision with+ its own indexed document and get re-ingested — the case that matters here is+ exactly the large container.+ """+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ # Truncated: the blob IS ingested, but the capped listing leaves it out.+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ remote_files = [{"id": "b::synced.pdf", "name": "synced.pdf", "modified_time": None}]+ service = _bucket_sync_service(remote_files)+ # The per-id lookup knows the truth the capped listing lost.+ session_manager, _ = _session_manager_finding("synced.pdf", synced_ids=("b::synced.pdf",))++ await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True+ ),+ request=MagicMock(),+ connector_service=service,+ session_manager=session_manager,+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )++ # Not a collision: it falls through to change detection, which sees a blob+ # the truncated listing calls "new" and ingests it plainly — no replace.+ replace_calls = [+ c for c in service.sync_specific_files.await_args_list if c.kwargs.get("replace_duplicates")+ ]+ assert replace_calls == []+++def _two_connection_service(remote_files, *, broken_first: bool = False):+ """Two active connections of one type, in list order conn-1, conn-2.++ connector_check_duplicates resolves the requested id, so the sync has to+ resolve the same one or the dialog and the ingest describe different+ connections.+ """+ connections = [+ SimpleNamespace(connection_id="conn-1", is_active=True),+ SimpleNamespace(connection_id="conn-2", is_active=True),+ ]+ connectors = {}+ for conn in connections:+ connector = _bucket_connector(remote_files)+ connector.authenticate = AsyncMock(+ return_value=not (broken_first and conn.connection_id == "conn-1")+ )+ connectors[conn.connection_id] = connector++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=connections)+ service.get_connector = AsyncMock(side_effect=lambda cid: connectors[cid])+ service.sync_specific_files = AsyncMock(return_value="task-x")+ return service+++async def _sync_with_connection(service, connection_id, monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))++ return await connectors_api.connector_sync(+ "ibm_cos",+ connectors_api.ConnectorSyncBody(connection_id=connection_id, bucket_filter=["b"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=_session_manager_finding()[0],+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),+ session=MagicMock(),+ rbac=_rbac_allowing("knowledge:delete:anonymous"),+ )+++async def test_sync_uses_the_requested_connection(monkeypatch):+ """Not merely the first that authenticates — conn-2 is second in list order."""+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]+ service = _two_connection_service(remote_files)++ await _sync_with_connection(service, "conn-2", monkeypatch)++ assert service.sync_specific_files.await_args.args[0] == "conn-2"+++async def test_sync_without_a_requested_connection_takes_the_first_working(monkeypatch):+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]+ service = _two_connection_service(remote_files)++ await _sync_with_connection(service, None, monkeypatch)++ assert service.sync_specific_files.await_args.args[0] == "conn-1"+++async def test_a_requested_connection_that_cannot_authenticate_falls_back(monkeypatch):+ """Ordering rather than hard selection: an unusable connection behaves as it+ did before, rather than failing later inside the connector."""+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]+ service = _two_connection_service(remote_files, broken_first=True)++ await _sync_with_connection(service, "conn-1", monkeypatch)++ assert service.sync_specific_files.await_args.args[0] == "conn-2"diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.pyindex 653d45c74..05f043f5b 100644--- a/tests/unit/api/test_connector_change_detection.py+++ b/tests/unit/api/test_connector_change_detection.py@@ -745,6 +745,10 @@ async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):assert new_call.kwargs.get("replace_duplicates", False) is Falseassert changed_call.args[2] == ["c::ingested_changed"]assert changed_call.kwargs["replace_duplicates"] is True+ # Each batch carries its own slice of the listing, so task rows can show+ # filenames instead of raw "<bucket>::<key>" ids.+ assert [f["id"] for f in new_call.kwargs["file_infos"]] == ["c::new"]+ assert [f["id"] for f in changed_call.kwargs["file_infos"]] == ["c::ingested_changed"]@pytest.mark.asynciodiff --git a/tests/unit/test_opensearch_filenames.py b/tests/unit/test_opensearch_filenames.pyindex b360a80fc..62449738c 100644--- a/tests/unit/test_opensearch_filenames.py+++ b/tests/unit/test_opensearch_filenames.py@@ -9,7 +9,11 @@ from unittest.mock import AsyncMockimport pytest-from utils.opensearch_filenames import filename_exists, find_existing_filenames+from utils.opensearch_filenames import (+ FILENAME_LOOKUP_BATCH_SIZE,+ filename_exists,+ find_existing_filenames,+)def _agg_response(*filenames: str) -> dict:@@ -72,3 +76,26 @@ async def test_filename_exists_blank_input_short_circuits():client = AsyncMock()assert await filename_exists("", client, "docs") is Falseclient.search.assert_not_awaited()+++async def test_find_existing_filenames_batches_large_candidate_sets():+ """A whole-container duplicate check can ask about more filenames than+ OpenSearch accepts in one terms query (index.max_terms_count, 65_536 by+ default), so candidates go out in batches and the results are unioned.+ """+ client = AsyncMock()+ candidates = [f"file-{i:05d}.pdf" for i in range(FILENAME_LOOKUP_BATCH_SIZE * 2 + 1)]++ async def search(*, index, body):+ asked = body["query"]["terms"]["filename"]+ assert len(asked) <= FILENAME_LOOKUP_BATCH_SIZE+ # One hit per batch, so the union has to come from every call.+ return _agg_response(asked[0])++ client.search = AsyncMock(side_effect=search)++ result = await find_existing_filenames(candidates, client, "docs")++ assert client.search.await_count == 3+ assert result == {"file-00000.pdf", "file-01024.pdf", "file-02048.pdf"}diff --git a/tests/unit/test_reconcile_shared_owner.py b/tests/unit/test_reconcile_shared_owner.pyindex 1e4471c46..99232a8c3 100644--- a/tests/unit/test_reconcile_shared_owner.py+++ b/tests/unit/test_reconcile_shared_owner.py@@ -88,3 +88,68 @@ async def test_reconcile_shared_owner_swallows_update_errors():# A failed reconciliation must not fail the (already-skipped) sync item.await processor._reconcile_shared_owner("report.pdf", True)+++async def test_reconcile_without_permission_leaves_a_shared_namesake_alone():+ """A private sync whose filename collides with someone else's SHARED+ document must not claim it.++ The script writes an owner onto whatever it matches, so the wider+ "mine OR ownerless" scope would hand a document visible to the whole+ instance to the syncing user — silently, and without the+ knowledge:delete:anonymous check that deliberate shared deletion goes+ through. _indexed_shared_state cannot catch this: it keys on the connector+ file id, and a collision comes from a document this connector never+ ingested, so `shared` falls back to the sync's own private intent.+ """+ from utils.opensearch_queries import build_owned_filename_query++ write_client = AsyncMock()+ settings_module.clients.opensearch = write_client++ processor = _make_processor(shared=False)+ processor.allow_anonymous_delete = False++ await processor._reconcile_shared_owner("report.pdf", False)++ query = write_client.update_by_query.await_args.kwargs["body"]["query"]+ assert query == build_owned_filename_query("report.pdf", "user-1")+++async def test_reconcile_with_permission_keeps_the_wider_scope():+ """Granted (and with RBAC unenforced, which resolves to granted), the+ reconcile still reaches ownerless chunks — the behaviour a shared re-sync+ relies on."""+ from utils.opensearch_queries import build_replace_filename_query++ write_client = AsyncMock()+ settings_module.clients.opensearch = write_client++ processor = _make_processor(shared=False)+ processor.allow_anonymous_delete = True++ await processor._reconcile_shared_owner("report.pdf", False)++ query = write_client.update_by_query.await_args.kwargs["body"]["query"]+ assert query == build_replace_filename_query("report.pdf", "user-1")+++async def test_reconcile_of_a_shared_file_keeps_the_wider_scope():+ """A shared write cleared the permission upstream — connector_sync rejects a+ shared sync without it — so resolving to shared keeps the wider scope even+ when the flag is False."""+ from utils.opensearch_queries import build_replace_filename_query++ write_client = AsyncMock()+ settings_module.clients.opensearch = write_client++ processor = _make_processor(shared=True)+ processor.allow_anonymous_delete = False++ await processor._reconcile_shared_owner("report.pdf", True)++ query = write_client.update_by_query.await_args.kwargs["body"]["query"]+ assert query == build_replace_filename_query("report.pdf", "user-1")diff --git a/tests/unit/test_traditional_duplicate_handling.py b/tests/unit/test_traditional_duplicate_handling.pyindex 608385aa6..5e378cc5c 100644--- a/tests/unit/test_traditional_duplicate_handling.py+++ b/tests/unit/test_traditional_duplicate_handling.py@@ -334,3 +334,98 @@ async def test_s3_processor_no_duplicate_proceeds():assert file_task.status == TaskStatus.COMPLETEDprocessor.delete_document_by_filename.assert_not_called()processor.process_document_standard.assert_awaited_once()+++def _delete_scope_env(monkeypatch):+ """Admin write client + a user client that reports one visible chunk."""+ from types import SimpleNamespace++ admin_client = AsyncMock()+ admin_client.delete = AsyncMock(return_value={"result": "deleted"})+ monkeypatch.setattr(+ "config.settings.clients",+ SimpleNamespace(opensearch=admin_client),+ )+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}+ )+ return opensearch_client+++async def test_replace_without_anonymous_delete_permission_stays_owner_scoped(monkeypatch):+ """Ownerless chunks are visible to the whole instance, so replacing a+ document that turns out to be one is a deletion of shared content. Without+ knowledge:delete:anonymous the scope must stay owner-only, leaving someone+ else's shared document alone."""+ from models.processors import TaskProcessor++ opensearch_client = _delete_scope_env(monkeypatch)++ await TaskProcessor().delete_document_by_filename(+ "report.pdf",+ opensearch_client,+ owner_user_id="user-123",+ shared=False,+ allow_anonymous_delete=False,+ )++ from utils.opensearch_queries import build_owned_filename_query++ query = opensearch_client.search.await_args.kwargs["body"]["query"]+ assert query == build_owned_filename_query("report.pdf", "user-123")+++async def test_shared_write_still_widens_without_the_flag(monkeypatch):+ """A shared write already required the permission upstream (connector_sync+ returns 403 without it), so the flag must not narrow that path."""+ from models.processors import TaskProcessor++ opensearch_client = _delete_scope_env(monkeypatch)++ await TaskProcessor().delete_document_by_filename(+ "report.pdf",+ opensearch_client,+ owner_user_id="user-123",+ shared=True,+ allow_anonymous_delete=False,+ )++ from utils.opensearch_queries import build_replace_filename_query++ query = opensearch_client.search.await_args.kwargs["body"]["query"]+ assert query == build_replace_filename_query("report.pdf", "user-123")+++async def test_resolve_duplicate_filename_forwards_the_permission(monkeypatch):+ """The gate every processor shares hands its caller's resolved permission to+ the delete, rather than deciding the scope on its own."""+ from models.processors import TaskProcessor++ processor = TaskProcessor()+ processor.check_filename_exists = AsyncMock(return_value=True)+ processor.delete_document_by_filename = AsyncMock(return_value=1)++ from types import SimpleNamespace++ monkeypatch.setattr(+ "models.processors.clients",+ SimpleNamespace(opensearch=AsyncMock()),+ )++ await processor.resolve_duplicate_filename(+ "report.pdf",+ AsyncMock(),+ replace=True,+ owner_user_id="user-123",+ allow_anonymous_delete=False,+ )++ assert (+ processor.delete_document_by_filename.await_args.kwargs["allow_anonymous_delete"] is False+ )