Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: detect cloud-connector duplicates ingested from other sources (#2404)

Rico Furtado9aa4aceba48f
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsx
index 2d5acd5a3..9f554ffe7 100644
--- a/frontend/components/connectors/shared-bucket-view.tsx
+++ b/frontend/components/connectors/shared-bucket-view.tsx
@@ -98,7 +98,6 @@ export function SharedBucketView({
const [pendingDuplicates, setPendingDuplicates] = useState<{
duplicateNames: string[];
duplicateCount: number;
- duplicateFiles: BucketDuplicateFile[];
nonDuplicateFiles: BucketDuplicateFile[];
} | null>(null);
const isOverwriteConfirmedRef = useRef(false);
@@ -177,9 +176,12 @@ export function SharedBucketView({
};
// Full bucket sync: backend classifies new/changed/unchanged itself and
- // re-ingests both new and changed blobs (used both when there are no
- // duplicates and when the user chooses to overwrite them).
- const runBucketSync = () => {
+ // re-ingests both new and changed blobs. With `replaceDuplicates`, it instead
+ // re-ingests the whole selection and replaces what is already indexed — the
+ // overwrite path. Either way the bucket names travel instead of the file
+ // list, so the request stays the same size for a 10-object bucket and a
+ // 10,000-object one.
+ const runBucketSync = (replaceDuplicates = false) => {
syncMutation.mutate(
{
connectorType: connector.type,
@@ -188,6 +190,7 @@ export function SharedBucketView({
selected_files: [],
bucket_filter: Array.from(selectedBuckets),
settings: ingestSettings,
+ replace_duplicates: replaceDuplicates,
shared: showSharedToggle
? (ingestSettings.shared ?? false)
: undefined,
@@ -197,15 +200,11 @@ export function SharedBucketView({
);
};
- // Sync explicit files. `replaceDuplicates` forces an unconditional
- // re-ingest (bypassing the bucket_filter path's modified-time gate) —
- // used when the user confirms "Overwrite duplicates" so an already
- // up-to-date file still gets re-ingested, matching what "overwrite" means
- // for the OAuth connectors.
- const runSelectedFilesSync = (
- files: BucketDuplicateFile[],
- replaceDuplicates = false,
- ) => {
+ // Sync an explicit subset of the listing — how "skip duplicates" ingests
+ // only the files that are not already indexed. (Overwrite goes through
+ // runBucketSync(true) instead: it needs the whole selection, and naming the
+ // buckets says that in one line.)
+ const runSelectedFilesSync = (files: BucketDuplicateFile[]) => {
syncMutation.mutate(
{
connectorType: connector.type,
@@ -213,7 +212,7 @@ export function SharedBucketView({
connection_id: connector.connectionId!,
selected_files: files,
settings: ingestSettings,
- replace_duplicates: replaceDuplicates,
+ replace_duplicates: false,
shared: showSharedToggle
? (ingestSettings.shared ?? false)
: undefined,
@@ -272,7 +271,6 @@ export function SharedBucketView({
setPendingDuplicates({
duplicateNames,
duplicateCount,
- duplicateFiles: checkData.duplicate_files || [],
nonDuplicateFiles: checkData.non_duplicate_files || [],
});
setDuplicateDialogOpen(true);
@@ -289,8 +287,7 @@ export function SharedBucketView({
const handleOverwriteDuplicates = () => {
if (!pendingDuplicates) return;
isOverwriteConfirmedRef.current = true;
- const { duplicateFiles, nonDuplicateFiles } = pendingDuplicates;
- runSelectedFilesSync([...duplicateFiles, ...nonDuplicateFiles], true);
+ runBucketSync(true);
setPendingDuplicates(null);
};
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 9afe02f56..9330de8ae 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -940,6 +940,7 @@ async def _sync_existing_connector_files(
ingest_settings=ingest_settings,
replace_duplicates=True,
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
else:
if max_files is not None:
@@ -952,6 +953,7 @@ async def _sync_existing_connector_files(
ingest_settings=ingest_settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
else:
# Fallback: use filename filtering (for Langflow-ingested files without document_id)
@@ -969,11 +971,18 @@ async def _sync_existing_connector_files(
ingest_settings=ingest_settings,
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
return {"outcome": "synced", "task_id": task_id}
class ConnectorSyncBody(BaseModel):
+ # The connection this sync targets. The upload UI is rendered per connection
+ # and sends it on every request; connector_check_duplicates resolves the same
+ # id, so honouring it here keeps the dialog and the sync it confirms pointed
+ # at one connection. Unset (or unknown) falls back to the first that
+ # authenticates, as before.
+ connection_id: str | None = None
max_files: int | None = None
selected_files: list[Any] | None = None
# When True, ingest ALL files from the connector (bypasses the existing-files gate).
@@ -985,7 +994,9 @@ class ConnectorSyncBody(BaseModel):
settings: dict[str, Any] | None = None
# When True, files whose filename already exists in the index are replaced
# rather than failing. Set by the provider upload UI after the user confirms
- # overwrite in the duplicate dialog.
+ # overwrite in the duplicate dialog. Honored for an explicit ``selected_files``
+ # sync and for a whole-container ``bucket_filter`` sync, where it also
+ # bypasses the modified-time/etag gate (see connector_sync).
replace_duplicates: bool = False
# When True (OSS only for now; SaaS deferred), run the ingest in preview mode
# (same as direct upload). Honored only when is_ingest_preview_enabled().
@@ -1141,6 +1152,161 @@ def _connector_scoped_to_buckets(connector, bucket_names: list[str]):
return scoped
+def _cleaned_blob_filename(file_info: dict[str, Any]) -> str:
+ """The name a remote blob would be indexed under, cleaned exactly as
+ ingestion cleans it — so name comparisons here mean what they say."""
+ from utils.file_utils import clean_connector_filename
+
+ return clean_connector_filename(
+ file_info.get("name", ""),
+ file_info.get("mimeType") or file_info.get("mime_type") or file_info.get("mimetype") or "",
+ )
+
+
+# Ids travel to OpenSearch in `terms` clauses, which are capped by
+# index.max_terms_count (65_536 by default). Whole-container syncs can exceed
+# that, so id lookups go out in batches.
+_ID_LOOKUP_BATCH_SIZE = 1024
+
+
+async def _bucket_blob_ids_already_synced(
+ file_ids: list[str],
+ connector_type: str,
+ session_manager,
+ user_id: str,
+ jwt_token: str | None,
+) -> set[str]:
+ """Which of exactly these blob ids already have chunks under this connector.
+
+ ``get_synced_file_ids_for_connector`` answers the same question for the
+ whole connector in one terms aggregation, which is capped at
+ ``OPENSEARCH_TERMS_AGG_LIMIT``: past the cap it omits ids that ARE ingested.
+ Reading "absent from that set" as "this connector never synced it" would
+ then turn an already-synced blob into a name collision — its own indexed
+ document is what holds the name — and re-ingest it under overwrite, in
+ exactly the large containers where the cap is reached. Asking about the
+ handful of ids actually in question keeps the answer independent of how much
+ the connector has ingested in total.
+
+ The id is matched in both fields it can live in, as
+ ``connectors.chunk_cleanup`` does: the standard ingest path stores it in
+ ``connector_file_id``, the Langflow path in ``document_id``.
+ """
+ ids = [fid for fid in file_ids if fid]
+ if not ids:
+ return set()
+
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+ synced: set[str] = set()
+
+ for start in range(0, len(ids), _ID_LOOKUP_BATCH_SIZE):
+ batch = ids[start : start + _ID_LOOKUP_BATCH_SIZE]
+ body: dict[str, Any] = {
+ "size": 0,
+ "query": {
+ "bool": {
+ "filter": [
+ {"term": {"connector_type": connector_type}},
+ {
+ "bool": {
+ "should": [
+ {"terms": {"document_id": batch}},
+ {"terms": {"connector_file_id": batch}},
+ # Indices predating the explicit keyword
+ # mapping dynamically mapped this as text.
+ {"terms": {"connector_file_id.keyword": batch}},
+ ],
+ "minimum_should_match": 1,
+ }
+ },
+ ]
+ }
+ },
+ "aggs": {
+ "connector_file_ids": {"terms": {"field": "connector_file_id", "size": len(batch)}},
+ "document_ids": {"terms": {"field": "document_id", "size": len(batch)}},
+ },
+ }
+
+ try:
+ response = await opensearch_client.search(index=get_index_name(), body=body)
+ except Exception as search_err:
+ if _is_unmapped_keyword_agg_error(search_err):
+ body["aggs"]["connector_file_ids"]["terms"]["field"] = "connector_file_id.keyword"
+ response = await opensearch_client.search(index=get_index_name(), body=body)
+ elif "index_not_found_exception" in str(search_err):
+ return set()
+ else:
+ raise
+
+ aggregations = response.get("aggregations", {})
+ for agg_name in ("connector_file_ids", "document_ids"):
+ for bucket in aggregations.get(agg_name, {}).get("buckets", []):
+ if bucket.get("key"):
+ synced.add(bucket["key"])
+
+ # Only ids we asked about; the aggregations can also surface an id that
+ # merely shares a chunk with one of them.
+ return synced & set(ids)
+
+
+async def _bucket_blob_ids_with_indexed_filename(
+ files: list[dict[str, Any]],
+ session_manager,
+ user_id: str,
+ jwt_token: str | None,
+) -> dict[str, str]:
+ """Map each blob whose filename already has indexed chunks — from any source,
+ a direct upload, another connector, or an earlier sync of this one — to the
+ indexed name it lands on.
+
+ The name, not just the id, because two blobs in one container can land on
+ the same one: a bucket connector names a blob after its key's basename, so
+ ``a/report.pdf`` and ``b/report.pdf`` are both ``report.pdf``. Callers that
+ act on the collision need to know which of them compete.
+
+ One batched terms aggregation for the whole listing, through the same
+ ``find_existing_filenames`` the OAuth connector classifier and the per-file
+ processor backstop use, so the duplicate question gets one answer at every
+ altitude. A missing index means nothing is indexed yet, not a failure.
+ """
+ from utils.file_utils import get_filename_aliases
+ from utils.opensearch_filenames import find_existing_filenames
+
+ aliases_by_id: dict[str, list[str]] = {}
+ all_candidates: set[str] = set()
+ for f in files:
+ fid = f.get("id")
+ if not fid:
+ continue
+ aliases = get_filename_aliases(_cleaned_blob_filename(f))
+ aliases_by_id[fid] = aliases
+ all_candidates.update(aliases)
+
+ if not all_candidates:
+ return {}
+
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+ try:
+ existing_filenames = await find_existing_filenames(
+ all_candidates, opensearch_client, get_index_name()
+ )
+ except Exception as search_err:
+ if "index_not_found_exception" not in str(search_err):
+ raise
+ return {}
+
+ # Listing order, so a caller resolving a contested name gets a stable winner
+ # (bucket listings are ordered by key).
+ matched: dict[str, str] = {}
+ for fid, aliases in aliases_by_id.items():
+ for alias in aliases:
+ if alias in existing_filenames:
+ matched[fid] = alias
+ break
+ return matched
+
+
async def _classify_bucket_connector_duplicates(
connector,
connector_type: str,
@@ -1149,13 +1315,14 @@ async def _classify_bucket_connector_duplicates(
user_id: str,
jwt_token: str | None,
) -> dict[str, Any]:
- """Preview a bucket_filter sync: classify remote blobs new/changed/unchanged
- without ingesting anything, mirroring the reconciliation in connector_sync's
- bucket_filter branch. "changed" blobs are reported as duplicates (they would
- overwrite an already-indexed version); "unchanged" blobs are silently
- dropped (the real sync would skip them too); "new" blobs are returned as
- ``non_duplicate_files`` so the caller can sync just those when the user
- chooses to skip duplicates.
+ """Preview a bucket_filter sync: classify remote blobs as duplicate or new
+ without ingesting anything, so the upload UI can confirm an overwrite.
+
+ A blob is a duplicate when it is already indexed under THIS connector type
+ (id match) or when its filename is already in the index from any other
+ source — a direct upload, a different connector. Blobs that match neither
+ are returned as ``non_duplicate_files`` so the caller can sync just those
+ when the user chooses to skip duplicates.
"""
scoped_connector = _connector_scoped_to_buckets(connector, bucket_filter)
all_files: list[dict[str, Any]] = []
@@ -1184,33 +1351,39 @@ async def _classify_bucket_connector_duplicates(
)
existing_set = set(existing_ids)
- # Existence-based, like the OAuth connector duplicate check: any blob
- # already ingested under this connector_type is a "duplicate" regardless
- # of whether the remote copy is newer. (The real bucket_filter sync uses
- # modified_time to auto-skip unchanged blobs on ITS OWN — that's a
- # separate, silent optimization; the confirm dialog here is about whether
- # the user wants to touch an already-indexed file at all, same as it
- # would for Google Drive/OneDrive/SharePoint.)
+ # The id half answers "did this connector already ingest this blob?"; the
+ # filename half answers "is this name already in the index at all?" — the
+ # question a direct upload, or an ingest through a different connector,
+ # makes relevant. Only the id half existed here, so a bucket whose blobs
+ # collide by name with files ingested elsewhere reported zero duplicates,
+ # skipped the confirm dialog, and then had every colliding file skipped at
+ # ingest time with "A file with this name already exists".
+ name_taken_ids = await _bucket_blob_ids_with_indexed_filename(
+ all_files, session_manager, user_id, jwt_token
+ )
+
duplicate_files: list[dict[str, Any]] = []
duplicate_names: list[str] = []
non_duplicate_files: list[dict[str, Any]] = []
+ total = 0
for f in all_files:
fid = f.get("id")
if not fid:
continue
- if fid in existing_set:
- response_file = _connector_file_response(f)
+ total += 1
+ response_file = _connector_file_response(f, cleaned_name=_cleaned_blob_filename(f))
+ if fid in existing_set or fid in name_taken_ids:
duplicate_files.append(response_file)
duplicate_names.append(response_file["name"])
else:
- non_duplicate_files.append(_connector_file_response(f))
+ non_duplicate_files.append(response_file)
return {
"duplicate_names": list(dict.fromkeys(duplicate_names)),
"duplicate_files": duplicate_files,
"non_duplicate_files": non_duplicate_files,
"duplicate_count": len(duplicate_files),
- "total_files": len(all_files),
+ "total_files": total,
}
@@ -1540,9 +1713,27 @@ async def connector_sync(
status_code=404,
)
+ # Try the requested connection first, then the rest. Ordering rather than
+ # selecting outright keeps today's behaviour when that connection cannot
+ # authenticate: the sync falls through to another, or reports that none
+ # work, instead of failing later inside the connector.
+ candidate_connections = active_connections
+ if body.connection_id:
+ requested = [c for c in active_connections if c.connection_id == body.connection_id]
+ if requested:
+ candidate_connections = requested + [
+ c for c in active_connections if c.connection_id != body.connection_id
+ ]
+ else:
+ logger.warning(
+ "Requested connection is not active for this connector type",
+ connector_type=connector_type,
+ connection_id=body.connection_id,
+ )
+
# Find the first connection that actually works
working_connection = None
- for connection in active_connections:
+ for connection in candidate_connections:
logger.debug(
"Testing connection authentication",
connection_id=connection.connection_id,
@@ -1629,6 +1820,7 @@ async def connector_sync(
replace_duplicates=body.replace_duplicates,
preview_mode=preview_mode,
shared=body.shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
elif body.sync_all or body.bucket_filter:
# Full ingest: discover and ingest all files (or files from specific buckets).
@@ -1665,6 +1857,23 @@ async def connector_sync(
status_code=200,
)
+ # Carry the listing's names into the task so its file rows read
+ # "report.pdf" rather than the raw "<bucket>::<key>" id, which is
+ # all the task has until each blob is fetched.
+ infos_by_id = {
+ f["id"]: _connector_file_response(f) for f in all_files if f.get("id")
+ }
+ if not infos_by_id:
+ # Nothing addressable: every listed blob came back without an
+ # id, so there is no sync to start.
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": "No files found in the selected buckets.",
+ },
+ status_code=200,
+ )
+
# Classify each remote blob as new / changed / unchanged.
existing_ids, _, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
@@ -1680,12 +1889,52 @@ async def connector_sync(
jwt_token=jwt_token,
)
+ # "Overwrite duplicates", confirmed in the UI dialog, is about
+ # blobs whose name is already indexed but which this connector
+ # has never synced — a direct upload, another connector. Nothing
+ # we store about them can say whether the source is newer, so
+ # change detection cannot decide them and they are replaced
+ # outright. It deliberately does NOT cover blobs this connector
+ # already synced: those keep the modified-time/etag gate, so
+ # confirming an overwrite re-ingests the colliding files rather
+ # than every object in the container.
+ colliding_ids: set[str] = set()
+ if body.replace_duplicates:
+ # One winner per contested name. Two blobs in the same
+ # container can land on the same indexed filename (a bucket
+ # connector names a blob after its key's basename, so
+ # a/report.pdf and b/report.pdf are both "report.pdf"), and
+ # files within a task are ingested concurrently — replacing
+ # one document from two of them at once has no defined
+ # winner. The first in listing order takes the name; the
+ # rest fall through to the per-file backstop, which skips
+ # them as duplicates exactly as it does today.
+ claimed_names: set[str] = set()
+ collisions = await _bucket_blob_ids_with_indexed_filename(
+ all_files, session_manager, user.user_id, jwt_token
+ )
+ # "Synced here already?" is asked of these ids directly
+ # rather than read off existing_set, whose aggregation is
+ # capped: above the cap an ingested blob is missing from it,
+ # and would look like a collision with its own document.
+ synced_collisions = await _bucket_blob_ids_already_synced(
+ list(collisions), connector_type, session_manager, user.user_id, jwt_token
+ )
+ for fid, indexed_name in collisions.items():
+ if fid in synced_collisions or indexed_name in claimed_names:
+ continue
+ claimed_names.add(indexed_name)
+ colliding_ids.add(fid)
+
new_ids: list[str] = []
- changed_ids: list[str] = []
+ replace_ids: list[str] = []
for f in all_files:
fid = f.get("id")
if not fid:
continue
+ if fid in colliding_ids:
+ replace_ids.append(fid)
+ continue
status = classify_remote_file_change(
fid,
f.get("modified_time"),
@@ -1696,7 +1945,7 @@ async def connector_sync(
if status == "new":
new_ids.append(fid)
elif status == "changed":
- changed_ids.append(fid)
+ replace_ids.append(fid)
# "unchanged" → skip; already ingested and not newer at source.
logger.info(
@@ -1704,11 +1953,12 @@ async def connector_sync(
connector_type=connector_type,
total=len(all_files),
new=len(new_ids),
- changed=len(changed_ids),
- skipped=len(all_files) - len(new_ids) - len(changed_ids),
+ replaced=len(replace_ids),
+ overwritten_duplicates=len(colliding_ids),
+ skipped=len(all_files) - len(new_ids) - len(replace_ids),
)
- if not new_ids and not changed_ids:
+ if not new_ids and not replace_ids:
return JSONResponse(
{
"status": "no_files",
@@ -1717,10 +1967,10 @@ async def connector_sync(
status_code=200,
)
- # Two batches: new files are created; changed files replace the
- # indexed copy (replace_duplicates=True bypasses the filename-skip
- # and deletes stale chunks before re-ingest). replace is batch-level,
- # hence the split.
+ # Two batches: new files are created; changed and overwritten
+ # files replace the indexed copy (replace_duplicates=True bypasses
+ # the filename-skip and deletes stale chunks before re-ingest).
+ # replace is batch-level, hence the split.
if new_ids:
task_ids.append(
await connector_service.sync_specific_files(
@@ -1728,22 +1978,26 @@ async def connector_sync(
user.user_id,
new_ids,
jwt_token=jwt_token,
+ file_infos=[infos_by_id[fid] for fid in new_ids],
ingest_settings=body.settings,
preview_mode=preview_mode,
shared=body.shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
)
- if changed_ids:
+ if replace_ids:
task_ids.append(
await connector_service.sync_specific_files(
working_connection.connection_id,
user.user_id,
- changed_ids,
+ replace_ids,
jwt_token=jwt_token,
+ file_infos=[infos_by_id[fid] for fid in replace_ids],
ingest_settings=body.settings,
replace_duplicates=True,
preview_mode=preview_mode,
shared=body.shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
)
else:
@@ -1755,6 +2009,7 @@ async def connector_sync(
jwt_token=jwt_token,
ingest_settings=body.settings,
shared=body.shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
else:
# No files specified - sync only files already in OpenSearch for this connector
diff --git a/src/connectors/service.py b/src/connectors/service.py
index c7bfa8de7..99ec51a3f 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -243,6 +243,7 @@ class ConnectorService:
ingest_settings: dict[str, Any] | None = None,
replace_duplicates: bool = False,
shared: bool | None = False,
+ allow_anonymous_delete: bool = True,
) -> str:
"""
Sync files from a connector connection using existing task tracking system.
@@ -352,6 +353,7 @@ class ConnectorService:
replace_duplicates=replace_duplicates,
connector_type=connector.CONNECTOR_TYPE,
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
# Use file IDs as items (no more fake file paths!)
@@ -385,6 +387,7 @@ class ConnectorService:
replace_duplicates: bool = False,
preview_mode: bool = False,
shared: bool | None = False,
+ allow_anonymous_delete: bool = True,
) -> str:
"""
Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
@@ -526,6 +529,7 @@ class ConnectorService:
replace_duplicates=replace_duplicates,
connector_type=connector.CONNECTOR_TYPE,
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
# Create custom task using TaskService
diff --git a/src/models/processors.py b/src/models/processors.py
index 77feb1a8f..bdee0021a 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -21,7 +21,7 @@ from utils.file_utils import (
)
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
-from utils.opensearch_queries import build_replace_filename_query
+from utils.opensearch_queries import build_owned_filename_query, build_replace_filename_query
from .tasks import FileTask, TaskStatus, UploadTask
@@ -259,6 +259,7 @@ class TaskProcessor:
replace: bool,
owner_user_id: str | None,
shared: bool = False,
+ allow_anonymous_delete: bool = True,
) -> Literal["proceed", "skip", "replaced"]:
"""Single duplicate-filename policy shared by every processor.
@@ -283,6 +284,7 @@ class TaskProcessor:
opensearch_client,
owner_user_id=owner_user_id,
shared=shared,
+ allow_anonymous_delete=allow_anonymous_delete,
)
if deleted == 0:
logger.warning(
@@ -322,22 +324,36 @@ class TaskProcessor:
opensearch_client,
owner_user_id: str | None = None,
shared: bool = False,
+ allow_anonymous_delete: bool = True,
) -> int:
"""Delete all chunks of a document with the given filename from
OpenSearch. Returns the number of chunks deleted.
``shared`` describes how the replacement is about to be *written*, not
what is already indexed, so it must not narrow what we delete: with an
- owner in hand the scope is always "owned by this user OR ownerless".
+ owner in hand the scope is "owned by this user OR ownerless".
Choosing an owner-only scope for a shared document matched none of its
chunks, and the caller read that zero as "nothing to replace" and
skipped the file — leaving the stale copy in the index even though the
duplicate check (which is owner-agnostic) had just found it and the user
- had confirmed the overwrite."""
+ had confirmed the overwrite.
+
+ ``allow_anonymous_delete`` is the caller's resolved
+ ``knowledge:delete:anonymous``. Ownerless chunks are visible to everyone
+ in the instance, so replacing a document that turns out to be one is a
+ deletion of shared content: without that permission the scope stays
+ owner-only and someone else's shared document is left alone (the file
+ then resolves as a duplicate the user may not replace). It defaults to
+ True because callers that have not resolved the permission — uploads,
+ the Langflow path, sample docs — keep their existing behaviour; see the
+ note in the PR about closing that across every entry point.
+ Deliberately ignored when ``shared`` is True: a shared write already
+ required the permission upstream."""
from config.settings import clients, get_index_name
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
from utils.opensearch_queries import (
build_anonymous_filename_query,
+ build_owned_filename_query,
build_replace_filename_query,
)
@@ -358,8 +374,10 @@ class TaskProcessor:
)
return 0
- else:
+ elif shared or allow_anonymous_delete:
build_query = build_replace_filename_query
+ else:
+ build_query = build_owned_filename_query
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
@@ -900,6 +918,7 @@ class ConnectorFileProcessor(TaskProcessor):
connector_type: str | None = None,
preview_mode: bool = False,
shared: bool | None = False,
+ allow_anonymous_delete: bool = True,
):
super().__init__(
document_service=document_service,
@@ -918,6 +937,10 @@ class ConnectorFileProcessor(TaskProcessor):
self.connector_type = connector_type
self.preview_mode = preview_mode
self.shared = shared
+ # The syncing user's resolved knowledge:delete:anonymous, threaded down
+ # to the replace path so an overwrite cannot delete a shared document
+ # the user is not allowed to delete.
+ self.allow_anonymous_delete = allow_anonymous_delete
async def _indexed_shared_state(
self,
@@ -995,9 +1018,20 @@ class ConnectorFileProcessor(TaskProcessor):
since then. Without this, those chunks would keep whatever owner they
got on their original ingest forever, since a byte-identical re-sync
never reaches resolve_shared_owner_fields(). Scoped to chunks owned by
- this user or already ownerless (matching the same boundary
- delete_document_by_filename uses), so it can't touch another user's
- private document that happens to share this filename.
+ this user — and to ownerless ones only under the same
+ ``knowledge:delete:anonymous`` boundary delete_document_by_filename
+ uses — so it can't touch another user's document that happens to share
+ this filename.
+
+ That boundary matters here even more than it does for a delete. An
+ ownerless document is visible to the whole instance, and this script
+ writes an owner onto what it matches: without the permission check, a
+ private sync of a file whose name collides with a shared document would
+ quietly claim that document for the syncing user, taking it out of
+ everyone else's view. ``_indexed_shared_state`` cannot prevent it — it
+ keys on the connector file id, and a collision by definition comes from
+ a document this connector never ingested, so the lookup returns None and
+ ``shared`` falls back to the sync's own (private) intent.
`shared` comes from _resolve_shared, never straight from self.shared: on
a re-sync it reflects the file's current indexed state, so this is a
@@ -1009,12 +1043,19 @@ class ConnectorFileProcessor(TaskProcessor):
owner, owner_name, owner_email = resolve_shared_owner_fields(
self.user_id, self.owner_name, self.owner_email, shared
)
+ # A shared write already cleared the permission upstream (connector_sync
+ # rejects shared syncs without it), so it keeps the wider scope.
+ build_query = (
+ build_replace_filename_query
+ if shared or self.allow_anonymous_delete
+ else build_owned_filename_query
+ )
for candidate in get_filename_aliases(filename):
try:
await write_client.update_by_query(
index=get_index_name(),
body={
- "query": build_replace_filename_query(candidate, self.user_id),
+ "query": build_query(candidate, self.user_id),
"script": {
"source": """
if (params.shared) {
@@ -1185,6 +1226,7 @@ class ConnectorFileProcessor(TaskProcessor):
replace=self.replace_duplicates,
owner_user_id=self.user_id,
shared=shared,
+ allow_anonymous_delete=self.allow_anonymous_delete,
)
if duplicate_action == "skip":
await self._reconcile_shared_owner(file_task.filename, shared)
diff --git a/src/utils/opensearch_filenames.py b/src/utils/opensearch_filenames.py
index 4f3b154a2..cfc1ad8f3 100644
--- a/src/utils/opensearch_filenames.py
+++ b/src/utils/opensearch_filenames.py
@@ -16,6 +16,12 @@ from collections.abc import Iterable
from utils.file_utils import get_filename_aliases
from utils.opensearch_queries import build_existing_filenames_agg_body
+# OpenSearch rejects a terms query carrying more than `index.max_terms_count`
+# values (default 65_536). Whole-container duplicate checks feed this function
+# one candidate per remote blob — times the alias expansion — so a large bucket
+# can exceed that on its own. Candidates go out in batches instead.
+FILENAME_LOOKUP_BATCH_SIZE = 1024
+
async def find_existing_filenames(
candidates: Iterable[str],
@@ -31,12 +37,16 @@ async def find_existing_filenames(
unique = sorted({c for c in candidates if c})
if not unique:
return set()
- response = await opensearch_client.search(
- index=index,
- body=build_existing_filenames_agg_body(unique),
- )
- buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])
- return {bucket["key"] for bucket in buckets if bucket.get("key")}
+ found: set[str] = set()
+ for start in range(0, len(unique), FILENAME_LOOKUP_BATCH_SIZE):
+ batch = unique[start : start + FILENAME_LOOKUP_BATCH_SIZE]
+ response = await opensearch_client.search(
+ index=index,
+ body=build_existing_filenames_agg_body(batch),
+ )
+ buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])
+ found.update(bucket["key"] for bucket in buckets if bucket.get("key"))
+ return found
async def filename_exists(
diff --git a/tests/unit/api/test_connector_bucket_duplicates.py b/tests/unit/api/test_connector_bucket_duplicates.py
new file mode 100644
index 000000000..8ece9c4d9
--- /dev/null
+++ b/tests/unit/api/test_connector_bucket_duplicates.py
@@ -0,0 +1,743 @@
+"""Duplicate handling for whole-container (bucket_filter) connector ingests.
+
+The bug these cover: the pre-check that decides whether the overwrite dialog
+appears was id-based only — "did THIS connector type already ingest this
+blob?" — while the per-file backstop in ConnectorFileProcessor is filename-based
+across every source. A bucket whose blobs share names with files ingested by
+direct upload therefore reported zero duplicates, skipped the dialog, and had
+every colliding file skipped at ingest time with "A file with this name already
+exists". The two altitudes now ask the same question, and an overwrite
+confirmed in the dialog reaches the bucket sync path.
+"""
+
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _json(response):
+ return json.loads(response.body.decode())
+
+
+def _rbac_allowing(*perms: str):
+ """RBAC stub granting exactly `perms` (unit tests run with enforcement on)."""
+ rbac = MagicMock()
+ granted = set(perms)
+
+ async def has_permission(user_id, perm, role_override=None):
+ return perm in granted
+
+ rbac.has_permission = AsyncMock(side_effect=has_permission)
+ return rbac
+
+
+def _session_manager_finding(*indexed_filenames: str, synced_ids: tuple[str, ...] = ()):
+ """Session manager whose OpenSearch client reports these filenames indexed,
+ and these blob ids already synced under the connector.
+
+ Serves both queries the sync issues for real — the filename terms
+ aggregation behind find_existing_filenames, and the per-id lookup that asks
+ which of those blobs this connector already ingested — so the real query
+ paths run rather than a stubbed-out classifier.
+ """
+ indexed = set(indexed_filenames)
+ synced = set(synced_ids)
+ client = AsyncMock()
+
+ async def search(*, index, body):
+ terms = body["query"].get("terms")
+ if terms is not None:
+ asked = terms["filename"]
+ return {
+ "aggregations": {
+ "filenames": {"buckets": [{"key": name} for name in asked if name in indexed]}
+ }
+ }
+
+ # The per-id "already synced here?" lookup.
+ should = body["query"]["bool"]["filter"][1]["bool"]["should"]
+ asked_ids = should[1]["terms"]["connector_file_id"]
+ return {
+ "aggregations": {
+ "connector_file_ids": {
+ "buckets": [{"key": fid} for fid in asked_ids if fid in synced]
+ },
+ "document_ids": {"buckets": []},
+ }
+ }
+
+ client.search = AsyncMock(side_effect=search)
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)
+ return session_manager, client
+
+
+def _bucket_connector(remote_files):
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+ connector.bucket_names = None
+ connector.list_files = AsyncMock(return_value={"files": remote_files, "next_page_token": None})
+ return connector
+
+
+def _bucket_sync_service(remote_files, task_id="task-x"):
+ connection = SimpleNamespace(connection_id="conn-1", is_active=True)
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])
+ service.get_connector = AsyncMock(return_value=_bucket_connector(remote_files))
+ service.sync_specific_files = AsyncMock(return_value=task_id)
+ return service
+
+
+# ---------------------------------------------------------------------------
+# _classify_bucket_connector_duplicates — what the confirm dialog is told
+# ---------------------------------------------------------------------------
+
+
+async def test_filename_indexed_by_another_source_is_a_duplicate(monkeypatch):
+ """The reported bug: sample.pdf was uploaded directly, so no COS id matches
+ it, but its name is taken. Without this the dialog never opened."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ session_manager, _ = _session_manager_finding("sample.pdf")
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=_bucket_connector(
+ [
+ {"id": "b::sample.pdf", "name": "sample.pdf"},
+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},
+ ]
+ ),
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="alice",
+ jwt_token="token",
+ )
+
+ assert result["duplicate_count"] == 1
+ assert result["duplicate_names"] == ["sample.pdf"]
+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::fresh.pdf"]
+ assert result["total_files"] == 2
+
+
+async def test_already_synced_id_is_still_a_duplicate(monkeypatch):
+ """The id half of the check survives: a blob this connector ingested before
+ counts even when its indexed chunks are invisible to the filename lookup."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["b::already.pdf"], [], "connector_file_id")),
+ )
+ session_manager, _ = _session_manager_finding()
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=_bucket_connector(
+ [
+ {"id": "b::already.pdf", "name": "already.pdf"},
+ {"id": "b::fresh.pdf", "name": "fresh.pdf"},
+ ]
+ ),
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="alice",
+ jwt_token="token",
+ )
+
+ assert result["duplicate_names"] == ["already.pdf"]
+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::fresh.pdf"]
+
+
+async def test_alias_match_counts_as_duplicate(monkeypatch):
+ """Legacy Langflow ingest indexes notes.txt as notes.md — the alias the
+ processor backstop also honors."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ session_manager, _ = _session_manager_finding("notes.md")
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=_bucket_connector([{"id": "b::notes.txt", "name": "notes.txt"}]),
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="alice",
+ jwt_token="token",
+ )
+
+ assert result["duplicate_names"] == ["notes.txt"]
+
+
+async def test_nothing_indexed_reports_no_duplicates(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ session_manager, _ = _session_manager_finding()
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=_bucket_connector([{"id": "b::a.pdf", "name": "a.pdf"}]),
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="alice",
+ jwt_token="token",
+ )
+
+ assert result["duplicate_count"] == 0
+ assert [f["id"] for f in result["non_duplicate_files"]] == ["b::a.pdf"]
+
+
+async def test_missing_index_is_not_an_error(monkeypatch):
+ """First ingest on a fresh instance: no documents index yet, nothing is a
+ duplicate. Any other search failure still propagates."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ client = AsyncMock()
+ client.search = AsyncMock(side_effect=Exception("index_not_found_exception: idx"))
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=client)
+
+ result = await connectors_api._classify_bucket_connector_duplicates(
+ connector=_bucket_connector([{"id": "b::a.pdf", "name": "a.pdf"}]),
+ connector_type="ibm_cos",
+ bucket_filter=["b"],
+ session_manager=session_manager,
+ user_id="alice",
+ jwt_token="token",
+ )
+
+ assert result["duplicate_count"] == 0
+
+
+# ---------------------------------------------------------------------------
+# connector_sync — what "overwrite" does to a whole-container selection
+# ---------------------------------------------------------------------------
+
+
+async def test_bucket_filter_overwrite_replaces_only_the_colliding_files(monkeypatch):
+ """Overwrite targets the blobs whose name is indexed but which this
+ connector never synced. Nothing we store about them can say whether the
+ source is newer, so change detection cannot decide them — they are replaced
+ outright, while a blob this connector already synced and that has not
+ changed is still skipped."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["b::synced.pdf"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={
+ "b::synced.pdf": connectors_api.SyncedFileState(
+ modified_time_ms=1704067200000.0, content_etag=None
+ )
+ }
+ ),
+ )
+
+ remote_files = [
+ # Uploaded directly before this connector existed: name taken, id new.
+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"},
+ # Synced by this connector and unchanged at source.
+ {"id": "b::synced.pdf", "name": "synced.pdf", "modified_time": "2024-01-01T00:00:00Z"},
+ # Not indexed anywhere.
+ {"id": "b::fresh.pdf", "name": "fresh.pdf", "modified_time": "2024-01-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+ session_manager, _ = _session_manager_finding(
+ "uploaded.pdf", "synced.pdf", synced_ids=("b::synced.pdf",)
+ )
+
+ response = await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ assert response.status_code == 201
+ assert service.sync_specific_files.await_count == 2
+ new_call, replace_call = service.sync_specific_files.await_args_list
+ assert new_call.args[2] == ["b::fresh.pdf"]
+ assert new_call.kwargs.get("replace_duplicates", False) is False
+ # Only the name collision is overwritten; the unchanged synced blob is not
+ # in either batch.
+ assert replace_call.args[2] == ["b::uploaded.pdf"]
+ assert replace_call.kwargs["replace_duplicates"] is True
+
+
+async def test_bucket_filter_overwrite_batches_collisions_with_changed_blobs(monkeypatch):
+ """A blob changed at source and a blob colliding by name both need the same
+ replace semantics, so they ride in one batch."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["b::changed.pdf"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_state_map",
+ AsyncMock(
+ return_value={
+ "b::changed.pdf": connectors_api.SyncedFileState(
+ modified_time_ms=1704067200000.0, content_etag=None
+ )
+ }
+ ),
+ )
+
+ remote_files = [
+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "b::changed.pdf", "name": "changed.pdf", "modified_time": "2024-06-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+ session_manager, _ = _session_manager_finding(
+ "uploaded.pdf", "changed.pdf", synced_ids=("b::changed.pdf",)
+ )
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ assert service.sync_specific_files.await_count == 1
+ call = service.sync_specific_files.await_args
+ assert sorted(call.args[2]) == ["b::changed.pdf", "b::uploaded.pdf"]
+ assert call.kwargs["replace_duplicates"] is True
+
+
+async def test_bucket_filter_without_overwrite_never_looks_up_filenames(monkeypatch):
+ """Declining the overwrite leaves the sync exactly as it was: the colliding
+ blob rides in the plain batch and the processor backstop skips it. No
+ filename lookup is issued."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [
+ {"id": "b::uploaded.pdf", "name": "uploaded.pdf", "modified_time": "2024-01-01T00:00:00Z"}
+ ]
+ service = _bucket_sync_service(remote_files)
+ session_manager, client = _session_manager_finding("uploaded.pdf")
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["b"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ client.search.assert_not_awaited()
+ call = service.sync_specific_files.await_args
+ assert call.args[2] == ["b::uploaded.pdf"]
+ assert call.kwargs.get("replace_duplicates", False) is False
+
+
+async def test_bucket_filter_sync_carries_filenames_into_the_task(monkeypatch):
+ """Without file_infos the task view shows the raw "<bucket>::<key>" id until
+ each blob is fetched."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [
+ {"id": "b::deep/report.pdf", "name": "report.pdf", "modified_time": None},
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["b"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ file_infos = service.sync_specific_files.await_args.kwargs["file_infos"]
+ assert [f["name"] for f in file_infos] == ["report.pdf"]
+ assert [f["id"] for f in file_infos] == ["b::deep/report.pdf"]
+
+
+async def test_bucket_filter_listing_without_ids_starts_no_sync(monkeypatch):
+ """Nothing addressable in the listing: report no_files rather than starting
+ an empty task."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ service = _bucket_sync_service([{"name": "no-id.pdf"}])
+
+ response = await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ assert response.status_code == 200
+ assert _json(response)["status"] == "no_files"
+ service.sync_specific_files.assert_not_awaited()
+
+
+async def test_overwrite_gives_one_winner_when_two_blobs_share_a_name(monkeypatch):
+ """A bucket connector names a blob after its key's basename, so a/report.pdf
+ and b/report.pdf both land on "report.pdf". Files within a task are ingested
+ concurrently, so letting both replace the same document has no defined
+ winner. The first in listing order takes the name; the other rides in the
+ plain batch, where the per-file backstop skips it as a duplicate.
+ """
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [
+ {"id": "b::a/report.pdf", "name": "report.pdf", "modified_time": None},
+ {"id": "b::c/report.pdf", "name": "report.pdf", "modified_time": None},
+ ]
+ service = _bucket_sync_service(remote_files)
+ session_manager, _ = _session_manager_finding("report.pdf")
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ assert service.sync_specific_files.await_count == 2
+ new_call, replace_call = service.sync_specific_files.await_args_list
+ assert replace_call.args[2] == ["b::a/report.pdf"]
+ assert replace_call.kwargs["replace_duplicates"] is True
+ assert new_call.args[2] == ["b::c/report.pdf"]
+ assert new_call.kwargs.get("replace_duplicates", False) is False
+
+
+async def test_alias_collision_between_two_blobs_gets_one_winner(monkeypatch):
+ """notes.txt and notes.md are ingestion aliases of one another, so two blobs
+ can contest a name without sharing one."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [
+ {"id": "b::notes.txt", "name": "notes.txt", "modified_time": None},
+ {"id": "b::notes.md", "name": "notes.md", "modified_time": None},
+ ]
+ service = _bucket_sync_service(remote_files)
+ session_manager, _ = _session_manager_finding("notes.md")
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ replace_calls = [
+ c for c in service.sync_specific_files.await_args_list if c.kwargs.get("replace_duplicates")
+ ]
+ assert len(replace_calls) == 1
+ assert replace_calls[0].args[2] == ["b::notes.txt"]
+
+
[email protected]("granted", [True, False])
+async def test_overwrite_carries_the_users_anonymous_delete_permission(monkeypatch, granted):
+ """Replacing a duplicate can delete a shared (ownerless) document, which is
+ what knowledge:delete:anonymous governs. connector_sync already resolves it
+ for orphan cleanup; the sync must hand the same answer to the replace path
+ rather than letting it widen on its own."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [{"id": "b::report.pdf", "name": "report.pdf", "modified_time": None}]
+ service = _bucket_sync_service(remote_files)
+ session_manager, _ = _session_manager_finding("report.pdf")
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous") if granted else _rbac_allowing(),
+ )
+
+ call = service.sync_specific_files.await_args
+ assert call.kwargs["replace_duplicates"] is True
+ assert call.kwargs["allow_anonymous_delete"] is granted
+
+
+async def test_overwrite_ignores_a_truncated_synced_id_listing(monkeypatch):
+ """get_synced_file_ids_for_connector's terms aggregation is capped at
+ OPENSEARCH_TERMS_AGG_LIMIT, so a container past the cap reports only some of
+ the ids it has ingested. An omitted blob must not become a collision with
+ its own indexed document and get re-ingested — the case that matters here is
+ exactly the large container.
+ """
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ # Truncated: the blob IS ingested, but the capped listing leaves it out.
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ remote_files = [{"id": "b::synced.pdf", "name": "synced.pdf", "modified_time": None}]
+ service = _bucket_sync_service(remote_files)
+ # The per-id lookup knows the truth the capped listing lost.
+ session_manager, _ = _session_manager_finding("synced.pdf", synced_ids=("b::synced.pdf",))
+
+ await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(
+ connection_id="conn-1", bucket_filter=["b"], replace_duplicates=True
+ ),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=session_manager,
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+ # Not a collision: it falls through to change detection, which sees a blob
+ # the truncated listing calls "new" and ingests it plainly — no replace.
+ replace_calls = [
+ c for c in service.sync_specific_files.await_args_list if c.kwargs.get("replace_duplicates")
+ ]
+ assert replace_calls == []
+
+
+def _two_connection_service(remote_files, *, broken_first: bool = False):
+ """Two active connections of one type, in list order conn-1, conn-2.
+
+ connector_check_duplicates resolves the requested id, so the sync has to
+ resolve the same one or the dialog and the ingest describe different
+ connections.
+ """
+ connections = [
+ SimpleNamespace(connection_id="conn-1", is_active=True),
+ SimpleNamespace(connection_id="conn-2", is_active=True),
+ ]
+ connectors = {}
+ for conn in connections:
+ connector = _bucket_connector(remote_files)
+ connector.authenticate = AsyncMock(
+ return_value=not (broken_first and conn.connection_id == "conn-1")
+ )
+ connectors[conn.connection_id] = connector
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=connections)
+ service.get_connector = AsyncMock(side_effect=lambda cid: connectors[cid])
+ service.sync_specific_files = AsyncMock(return_value="task-x")
+ return service
+
+
+async def _sync_with_connection(service, connection_id, monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(connectors_api, "get_synced_file_state_map", AsyncMock(return_value={}))
+
+ return await connectors_api.connector_sync(
+ "ibm_cos",
+ connectors_api.ConnectorSyncBody(connection_id=connection_id, bucket_filter=["b"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=_session_manager_finding()[0],
+ user=SimpleNamespace(user_id="alice", jwt_token="token", db_user_id="alice"),
+ session=MagicMock(),
+ rbac=_rbac_allowing("knowledge:delete:anonymous"),
+ )
+
+
+async def test_sync_uses_the_requested_connection(monkeypatch):
+ """Not merely the first that authenticates — conn-2 is second in list order."""
+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]
+ service = _two_connection_service(remote_files)
+
+ await _sync_with_connection(service, "conn-2", monkeypatch)
+
+ assert service.sync_specific_files.await_args.args[0] == "conn-2"
+
+
+async def test_sync_without_a_requested_connection_takes_the_first_working(monkeypatch):
+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]
+ service = _two_connection_service(remote_files)
+
+ await _sync_with_connection(service, None, monkeypatch)
+
+ assert service.sync_specific_files.await_args.args[0] == "conn-1"
+
+
+async def test_a_requested_connection_that_cannot_authenticate_falls_back(monkeypatch):
+ """Ordering rather than hard selection: an unusable connection behaves as it
+ did before, rather than failing later inside the connector."""
+ remote_files = [{"id": "b::a.pdf", "name": "a.pdf", "modified_time": None}]
+ service = _two_connection_service(remote_files, broken_first=True)
+
+ await _sync_with_connection(service, "conn-1", monkeypatch)
+
+ assert service.sync_specific_files.await_args.args[0] == "conn-2"
diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.py
index 653d45c74..05f043f5b 100644
--- a/tests/unit/api/test_connector_change_detection.py
+++ b/tests/unit/api/test_connector_change_detection.py
@@ -745,6 +745,10 @@ async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):
assert new_call.kwargs.get("replace_duplicates", False) is False
assert changed_call.args[2] == ["c::ingested_changed"]
assert changed_call.kwargs["replace_duplicates"] is True
+ # Each batch carries its own slice of the listing, so task rows can show
+ # filenames instead of raw "<bucket>::<key>" ids.
+ assert [f["id"] for f in new_call.kwargs["file_infos"]] == ["c::new"]
+ assert [f["id"] for f in changed_call.kwargs["file_infos"]] == ["c::ingested_changed"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_opensearch_filenames.py b/tests/unit/test_opensearch_filenames.py
index b360a80fc..62449738c 100644
--- a/tests/unit/test_opensearch_filenames.py
+++ b/tests/unit/test_opensearch_filenames.py
@@ -9,7 +9,11 @@ from unittest.mock import AsyncMock
import pytest
-from utils.opensearch_filenames import filename_exists, find_existing_filenames
+from utils.opensearch_filenames import (
+ FILENAME_LOOKUP_BATCH_SIZE,
+ filename_exists,
+ find_existing_filenames,
+)
def _agg_response(*filenames: str) -> dict:
@@ -72,3 +76,26 @@ async def test_filename_exists_blank_input_short_circuits():
client = AsyncMock()
assert await filename_exists("", client, "docs") is False
client.search.assert_not_awaited()
+
+
+async def test_find_existing_filenames_batches_large_candidate_sets():
+ """A whole-container duplicate check can ask about more filenames than
+ OpenSearch accepts in one terms query (index.max_terms_count, 65_536 by
+ default), so candidates go out in batches and the results are unioned.
+ """
+ client = AsyncMock()
+ candidates = [f"file-{i:05d}.pdf" for i in range(FILENAME_LOOKUP_BATCH_SIZE * 2 + 1)]
+
+ async def search(*, index, body):
+ asked = body["query"]["terms"]["filename"]
+ assert len(asked) <= FILENAME_LOOKUP_BATCH_SIZE
+ # One hit per batch, so the union has to come from every call.
+ return _agg_response(asked[0])
+
+ client.search = AsyncMock(side_effect=search)
+
+ result = await find_existing_filenames(candidates, client, "docs")
+
+ assert client.search.await_count == 3
+ assert result == {"file-00000.pdf", "file-01024.pdf", "file-02048.pdf"}
diff --git a/tests/unit/test_reconcile_shared_owner.py b/tests/unit/test_reconcile_shared_owner.py
index 1e4471c46..99232a8c3 100644
--- a/tests/unit/test_reconcile_shared_owner.py
+++ b/tests/unit/test_reconcile_shared_owner.py
@@ -88,3 +88,68 @@ async def test_reconcile_shared_owner_swallows_update_errors():
# A failed reconciliation must not fail the (already-skipped) sync item.
await processor._reconcile_shared_owner("report.pdf", True)
+
+
+async def test_reconcile_without_permission_leaves_a_shared_namesake_alone():
+ """A private sync whose filename collides with someone else's SHARED
+ document must not claim it.
+
+ The script writes an owner onto whatever it matches, so the wider
+ "mine OR ownerless" scope would hand a document visible to the whole
+ instance to the syncing user — silently, and without the
+ knowledge:delete:anonymous check that deliberate shared deletion goes
+ through. _indexed_shared_state cannot catch this: it keys on the connector
+ file id, and a collision comes from a document this connector never
+ ingested, so `shared` falls back to the sync's own private intent.
+ """
+ from utils.opensearch_queries import build_owned_filename_query
+
+ write_client = AsyncMock()
+ settings_module.clients.opensearch = write_client
+
+ processor = _make_processor(shared=False)
+ processor.allow_anonymous_delete = False
+
+ await processor._reconcile_shared_owner("report.pdf", False)
+
+ query = write_client.update_by_query.await_args.kwargs["body"]["query"]
+ assert query == build_owned_filename_query("report.pdf", "user-1")
+
+
+async def test_reconcile_with_permission_keeps_the_wider_scope():
+ """Granted (and with RBAC unenforced, which resolves to granted), the
+ reconcile still reaches ownerless chunks — the behaviour a shared re-sync
+ relies on."""
+ from utils.opensearch_queries import build_replace_filename_query
+
+ write_client = AsyncMock()
+ settings_module.clients.opensearch = write_client
+
+ processor = _make_processor(shared=False)
+ processor.allow_anonymous_delete = True
+
+ await processor._reconcile_shared_owner("report.pdf", False)
+
+ query = write_client.update_by_query.await_args.kwargs["body"]["query"]
+ assert query == build_replace_filename_query("report.pdf", "user-1")
+
+
+async def test_reconcile_of_a_shared_file_keeps_the_wider_scope():
+ """A shared write cleared the permission upstream — connector_sync rejects a
+ shared sync without it — so resolving to shared keeps the wider scope even
+ when the flag is False."""
+ from utils.opensearch_queries import build_replace_filename_query
+
+ write_client = AsyncMock()
+ settings_module.clients.opensearch = write_client
+
+ processor = _make_processor(shared=True)
+ processor.allow_anonymous_delete = False
+
+ await processor._reconcile_shared_owner("report.pdf", True)
+
+ query = write_client.update_by_query.await_args.kwargs["body"]["query"]
+ assert query == build_replace_filename_query("report.pdf", "user-1")
diff --git a/tests/unit/test_traditional_duplicate_handling.py b/tests/unit/test_traditional_duplicate_handling.py
index 608385aa6..5e378cc5c 100644
--- a/tests/unit/test_traditional_duplicate_handling.py
+++ b/tests/unit/test_traditional_duplicate_handling.py
@@ -334,3 +334,98 @@ async def test_s3_processor_no_duplicate_proceeds():
assert file_task.status == TaskStatus.COMPLETED
processor.delete_document_by_filename.assert_not_called()
processor.process_document_standard.assert_awaited_once()
+
+
+def _delete_scope_env(monkeypatch):
+ """Admin write client + a user client that reports one visible chunk."""
+ from types import SimpleNamespace
+
+ admin_client = AsyncMock()
+ admin_client.delete = AsyncMock(return_value={"result": "deleted"})
+ monkeypatch.setattr(
+ "config.settings.clients",
+ SimpleNamespace(opensearch=admin_client),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
+ )
+ return opensearch_client
+
+
+async def test_replace_without_anonymous_delete_permission_stays_owner_scoped(monkeypatch):
+ """Ownerless chunks are visible to the whole instance, so replacing a
+ document that turns out to be one is a deletion of shared content. Without
+ knowledge:delete:anonymous the scope must stay owner-only, leaving someone
+ else's shared document alone."""
+ from models.processors import TaskProcessor
+
+ opensearch_client = _delete_scope_env(monkeypatch)
+
+ await TaskProcessor().delete_document_by_filename(
+ "report.pdf",
+ opensearch_client,
+ owner_user_id="user-123",
+ shared=False,
+ allow_anonymous_delete=False,
+ )
+
+ from utils.opensearch_queries import build_owned_filename_query
+
+ query = opensearch_client.search.await_args.kwargs["body"]["query"]
+ assert query == build_owned_filename_query("report.pdf", "user-123")
+
+
+async def test_shared_write_still_widens_without_the_flag(monkeypatch):
+ """A shared write already required the permission upstream (connector_sync
+ returns 403 without it), so the flag must not narrow that path."""
+ from models.processors import TaskProcessor
+
+ opensearch_client = _delete_scope_env(monkeypatch)
+
+ await TaskProcessor().delete_document_by_filename(
+ "report.pdf",
+ opensearch_client,
+ owner_user_id="user-123",
+ shared=True,
+ allow_anonymous_delete=False,
+ )
+
+ from utils.opensearch_queries import build_replace_filename_query
+
+ query = opensearch_client.search.await_args.kwargs["body"]["query"]
+ assert query == build_replace_filename_query("report.pdf", "user-123")
+
+
+async def test_resolve_duplicate_filename_forwards_the_permission(monkeypatch):
+ """The gate every processor shares hands its caller's resolved permission to
+ the delete, rather than deciding the scope on its own."""
+ from models.processors import TaskProcessor
+
+ processor = TaskProcessor()
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ processor.delete_document_by_filename = AsyncMock(return_value=1)
+
+ from types import SimpleNamespace
+
+ monkeypatch.setattr(
+ "models.processors.clients",
+ SimpleNamespace(opensearch=AsyncMock()),
+ )
+
+ await processor.resolve_duplicate_filename(
+ "report.pdf",
+ AsyncMock(),
+ replace=True,
+ owner_user_id="user-123",
+ allow_anonymous_delete=False,
+ )
+
+ assert (
+ processor.delete_document_by_filename.await_args.kwargs["allow_anonymous_delete"] is False
+ )