OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: user-facing duplicate handling workflow for bucket-based connectors (backport of #2077) (#2096)
Edwin Josefe9425d02294
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex c4a6800c6..a549c2aa3 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -8,12 +8,28 @@ import type { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";import { IngestSettings } from "@/components/cloud-picker/ingest-settings";import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";+import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog";import { FileBrowserDialog } from "@/components/file-browser-dialog";import { Button } from "@/components/ui/button";import { useAuth } from "@/contexts/auth-context";import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";+interface BucketDuplicateFile {+ id: string;+ name: string;+ mimeType: string;+ downloadUrl?: string;+ size?: number;+}++interface BucketDuplicateCheckResponse {+ duplicate_names?: string[];+ duplicate_count?: number;+ duplicate_files?: BucketDuplicateFile[];+ non_duplicate_files?: BucketDuplicateFile[];+}+export interface SharedBucketViewProps {connector: any;buckets: Array<{ name: string; ingested_count: number }> | undefined;@@ -70,6 +86,15 @@ export function SharedBucketView({const [browseDialogBucket, setBrowseDialogBucket] = useState<string | null>(null,);+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);+ const [pendingDuplicates, setPendingDuplicates] = useState<{+ duplicateNames: string[];+ duplicateCount: number;+ duplicateFiles: BucketDuplicateFile[];+ nonDuplicateFiles: BucketDuplicateFile[];+ } | null>(null);+ const isOverwriteConfirmedRef = useRef(false);useEffect(() => {if (@@ -111,20 +136,43 @@ export function SharedBucketView({});};- const ingestSelected = () => {- const chunkErr = getIngestChunkSettingsError(ingestSettings);- if (chunkErr) {- toast.error("Could not start ingest", { description: chunkErr });- return;- }- trackStartProcess({+ const onSyncSettled = () => {+ invalidate();+ };++ const onSyncError = (err: unknown) => {+ trackProcessFailure({processType: "Ingestion",process: "Document Upload",category: "Knowledge",source: "connector",connector_type: connector.type,- total_buckets: selectedBuckets.size,+ resultValue: err instanceof Error ? err.message : "Sync failed",});+ toast.error(err instanceof Error ? err.message : "Sync failed");+ };++ const onSyncSuccess = (result: { task_ids?: string[]; message?: string }) => {+ onSyncSettled();+ if (result.task_ids?.length) {+ // The container path may return two tasks (new files + changed files);+ // track them all.+ for (const id of result.task_ids) {+ addTask(id, {+ connectorType: connector.type,+ source: "connector",+ });+ }+ onDone();+ } else {+ toast.info(result.message ?? "No files found in the selected buckets.");+ }+ };++ // Full bucket sync: backend classifies new/changed/unchanged itself and+ // re-ingests both new and changed blobs (used both when there are no+ // duplicates and when the user chooses to overwrite them).+ const runBucketSync = () => {syncMutation.mutate({connectorType: connector.type,@@ -138,35 +186,129 @@ export function SharedBucketView({: undefined,},},+ { onSuccess: onSyncSuccess, onError: onSyncError },+ );+ };++ // Sync explicit files. `replaceDuplicates` forces an unconditional+ // re-ingest (bypassing the bucket_filter path's modified-time gate) —+ // used when the user confirms "Overwrite duplicates" so an already+ // up-to-date file still gets re-ingested, matching what "overwrite" means+ // for the OAuth connectors.+ const runSelectedFilesSync = (+ files: BucketDuplicateFile[],+ replaceDuplicates = false,+ ) => {+ syncMutation.mutate({- onSuccess: (result) => {- invalidate();- if (result.task_ids?.length) {- // The container path may return two tasks (new files + changed files);- // track them all.- trackIngestTasks(result.task_ids);- onDone();- } else {- toast.info(- result.message ?? "No files found in the selected buckets.",- );- }- },- onError: (err) => {- trackProcessFailure({- processType: "Ingestion",- process: "Document Upload",- category: "Knowledge",- source: "connector",- connector_type: connector.type,- resultValue: err instanceof Error ? err.message : "Sync failed",- });- toast.error(err instanceof Error ? err.message : "Sync failed");+ connectorType: connector.type,+ body: {+ connection_id: connector.connectionId!,+ selected_files: files,+ settings: ingestSettings,+ replace_duplicates: replaceDuplicates,+ shared: showSharedToggle+ ? (ingestSettings.shared ?? false)+ : undefined,},},+ { onSuccess: onSyncSuccess, onError: onSyncError },);};+ const ingestSelected = async () => {+ const chunkErr = getIngestChunkSettingsError(ingestSettings);+ if (chunkErr) {+ toast.error("Could not start ingest", { description: chunkErr });+ return;+ }+ trackStartProcess({+ processType: "Ingestion",+ process: "Document Upload",+ category: "Knowledge",+ source: "connector",+ connector_type: connector.type,+ total_buckets: selectedBuckets.size,+ });++ setIsCheckingDuplicates(true);+ try {+ const checkResponse = await fetch(+ `/api/connectors/${connector.type}/check-duplicates`,+ {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ connection_id: connector.connectionId,+ bucket_filter: Array.from(selectedBuckets),+ }),+ },+ );++ if (!checkResponse.ok) {+ throw new Error(`Duplicate check failed: ${checkResponse.statusText}`);+ }++ const checkData =+ (await checkResponse.json()) as BucketDuplicateCheckResponse;+ const duplicateNames = checkData.duplicate_names || [];+ const duplicateCount =+ typeof checkData.duplicate_count === "number"+ ? checkData.duplicate_count+ : duplicateNames.length;++ if (duplicateCount === 0) {+ runBucketSync();+ return;+ }++ setPendingDuplicates({+ duplicateNames,+ duplicateCount,+ duplicateFiles: checkData.duplicate_files || [],+ nonDuplicateFiles: checkData.non_duplicate_files || [],+ });+ setDuplicateDialogOpen(true);+ } catch (err) {+ console.error("[Bucket Sync] Duplicate check failed:", err);+ // Fallback: proceed with the normal full sync (backend still handles+ // new/changed reconciliation on its own).+ runBucketSync();+ } finally {+ setIsCheckingDuplicates(false);+ }+ };++ const handleOverwriteDuplicates = () => {+ if (!pendingDuplicates) return;+ isOverwriteConfirmedRef.current = true;+ const { duplicateFiles, nonDuplicateFiles } = pendingDuplicates;+ runSelectedFilesSync([...duplicateFiles, ...nonDuplicateFiles], true);+ setPendingDuplicates(null);+ };++ const handleDuplicateDialogOpenChange = (open: boolean) => {+ if (!open && pendingDuplicates) {+ if (isOverwriteConfirmedRef.current) {+ // Overwrite already submitted in handleOverwriteDuplicates; this close+ // event fires immediately after and would otherwise re-enter the+ // "skip duplicates" branch.+ isOverwriteConfirmedRef.current = false;+ } else {+ const { nonDuplicateFiles, duplicateCount } = pendingDuplicates;+ if (nonDuplicateFiles.length > 0) {+ runSelectedFilesSync(nonDuplicateFiles);+ } else {+ toast.info(+ `All ${duplicateCount} file(s) already exist and were skipped. Nothing was synced.`,+ );+ }+ }+ setPendingDuplicates(null);+ }+ setDuplicateDialogOpen(open);+ };+return (<><div className="mb-8 flex gap-2 items-center">@@ -330,14 +472,20 @@ export function SharedBucketView({<ButtonclassName="bg-foreground text-background hover:bg-foreground/90 font-semibold"onClick={ingestSelected}- disabled={syncMutation.isPending || selectedBuckets.size === 0}- loading={syncMutation.isPending}+ disabled={+ syncMutation.isPending ||+ isCheckingDuplicates ||+ selectedBuckets.size === 0+ }+ loading={syncMutation.isPending || isCheckingDuplicates}>{syncMutation.isPending? "Ingesting…"- : selectedBuckets.size > 0- ? `Ingest ${selectedBuckets.size} Bucket${selectedBuckets.size !== 1 ? "s" : ""}`- : "Select Buckets to Ingest"}+ : isCheckingDuplicates+ ? "Checking…"+ : selectedBuckets.size > 0+ ? `Ingest ${selectedBuckets.size} Bucket${selectedBuckets.size !== 1 ? "s" : ""}`+ : "Select Buckets to Ingest"}</Button></div></div>@@ -366,6 +514,15 @@ export function SharedBucketView({ingestSettings={ingestSettings}/>)}++ <DuplicateHandlingDialog+ open={duplicateDialogOpen}+ onOpenChange={handleDuplicateDialogOpenChange}+ onOverwrite={handleOverwriteDuplicates}+ isLoading={syncMutation.isPending}+ duplicateNames={pendingDuplicates?.duplicateNames}+ duplicateCount={pendingDuplicates?.duplicateCount}+ /></>);}diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsxindex 1e89424b5..84149eed9 100644--- a/frontend/components/file-browser-dialog.tsx+++ b/frontend/components/file-browser-dialog.tsx@@ -1,7 +1,7 @@"use client";import { Check, Download, Loader2, RefreshCw, Search } from "lucide-react";-import { useCallback, useMemo, useState } from "react";+import { useCallback, useMemo, useRef, useState } from "react";import { toast } from "sonner";import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";import {@@ -10,6 +10,7 @@ import {} from "@/app/api/queries/useBrowseConnectionFiles";import type { IngestSettings } from "@/components/cloud-picker/types";import { formatFileSize } from "@/lib/file-format";+import { DuplicateHandlingDialog } from "./duplicate-handling-dialog";import { Badge } from "./ui/badge";import { Button } from "./ui/button";import {@@ -23,6 +24,19 @@ import {import { Input } from "./ui/input";import { ScrollArea } from "./ui/scroll-area";+interface SyncFilePayload {+ id: string;+ name: string;+ mimeType: string;+ size?: number;+}++interface FileBrowserDuplicateCheckResponse {+ duplicate_names?: string[];+ duplicate_count?: number;+ non_duplicate_files?: SyncFilePayload[];+}+interface FileBrowserDialogProps {open: boolean;onOpenChange: (open: boolean) => void;@@ -80,12 +94,10 @@ export function FileBrowserDialog({}, [allFiles, search]);// Selectable files within the current (filtered) view — drives "Select all".- // A file is selectable when it's not yet ingested, or it's ingested but stale- // (a newer version exists at the source, so re-ingest is allowed).- const visibleSelectable = useMemo(- () => files.filter((f) => !f.is_ingested || f.is_stale),- [files],- );+ // Already-ingested files are selectable too (re-checking one is an explicit+ // request to re-ingest it — handleIngest routes through the duplicate-check+ // confirmation flow before overwriting).+ const visibleSelectable = files;const allVisibleSelected =visibleSelectable.length > 0 &&@@ -127,171 +139,285 @@ export function FileBrowserDialog({[allFiles, selectedFileIds],);+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);+ const [pendingSync, setPendingSync] = useState<{+ allFiles: SyncFilePayload[];+ nonDuplicateFiles: SyncFilePayload[];+ duplicateNames: string[];+ duplicateCount: number;+ } | null>(null);+ const isOverwriteConfirmedRef = useRef(false);++ const submitSync = useCallback(+ async (syncFiles: SyncFilePayload[], replaceDuplicates: boolean) => {+ try {+ const result = await syncMutation.mutateAsync({+ connectorType,+ body: {+ selected_files: syncFiles,+ settings: ingestSettings,+ shared: showShared ? (ingestSettings?.shared ?? false) : undefined,+ replace_duplicates: replaceDuplicates,+ },+ });++ toast.success("Ingestion started", {+ description: `${syncFiles.length} file(s) queued for ingestion.`,+ });++ setSelectedFileIds(new Set());+ onOpenChange(false);+ onIngestSuccess?.(result);+ } catch (err) {+ toast.error("Ingestion failed", {+ description: err instanceof Error ? err.message : "Unknown error",+ });+ }+ },+ [+ connectorType,+ syncMutation,+ onOpenChange,+ onIngestSuccess,+ showShared,+ ingestSettings,+ ],+ );+const handleIngest = useCallback(async () => {if (selectedFiles.length === 0) return;- // If any selected file is a stale re-ingest, replace the indexed copy so the- // newer version overwrites the old chunks (rather than being skipped as a- // duplicate filename). New files are unaffected by this flag.+ const filesPayload: SyncFilePayload[] = selectedFiles.map((f) => ({+ id: f.id,+ name: f.name,+ mimeType: "",+ size: f.size,+ }));++ // If any selected file is a stale re-ingest, default to replacing the+ // indexed copy so the newer version overwrites the old chunks even if+ // the duplicate check below is skipped (network error) or doesn't flag+ // it. New files are unaffected by this flag.const hasStale = selectedFiles.some((f) => f.is_stale);+ setIsCheckingDuplicates(true);try {- const result = await syncMutation.mutateAsync({- connectorType,- body: {- selected_files: selectedFiles.map((f) => ({- id: f.id,- name: f.name,- mimeType: "",- size: f.size,- })),- settings: ingestSettings,- shared: showShared ? (ingestSettings?.shared ?? false) : undefined,- ...(hasStale ? { replace_duplicates: true } : {}),+ const checkResponse = await fetch(+ `/api/connectors/${connectorType}/check-duplicates`,+ {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ connection_id: connectionId,+ selected_files: filesPayload,+ }),},- });+ );- toast.success("Ingestion started", {- description: `${selectedFiles.length} file(s) queued for ingestion.`,- });+ if (!checkResponse.ok) {+ throw new Error(`Duplicate check failed: ${checkResponse.statusText}`);+ }- setSelectedFileIds(new Set());- onOpenChange(false);- onIngestSuccess?.(result);- } catch (err) {- toast.error("Ingestion failed", {- description: err instanceof Error ? err.message : "Unknown error",+ const checkData =+ (await checkResponse.json()) as FileBrowserDuplicateCheckResponse;+ const duplicateNames = checkData.duplicate_names || [];+ const duplicateCount =+ typeof checkData.duplicate_count === "number"+ ? checkData.duplicate_count+ : duplicateNames.length;++ if (duplicateCount === 0) {+ await submitSync(filesPayload, hasStale);+ return;+ }++ setPendingSync({+ allFiles: filesPayload,+ nonDuplicateFiles: checkData.non_duplicate_files || [],+ duplicateNames,+ duplicateCount,});+ setDuplicateDialogOpen(true);+ } catch (err) {+ console.error("[File Browser] Duplicate check failed:", err);+ // Fallback: proceed without overwrite (backend will still skip+ // exact-duplicate filenames on its own), unless we already know some+ // selected files are stale re-ingests.+ await submitSync(filesPayload, hasStale);+ } finally {+ setIsCheckingDuplicates(false);+ }+ }, [selectedFiles, connectorType, connectionId, submitSync]);++ const handleOverwriteDuplicates = () => {+ if (!pendingSync) return;+ isOverwriteConfirmedRef.current = true;+ submitSync(pendingSync.allFiles, true);+ setPendingSync(null);+ };++ const handleDuplicateDialogOpenChange = (open: boolean) => {+ if (!open && pendingSync) {+ if (isOverwriteConfirmedRef.current) {+ // Overwrite already submitted in handleOverwriteDuplicates; this close+ // event fires immediately after and would otherwise re-enter the+ // "skip duplicates" branch.+ isOverwriteConfirmedRef.current = false;+ } else {+ const { nonDuplicateFiles, duplicateCount } = pendingSync;+ if (nonDuplicateFiles.length > 0) {+ submitSync(nonDuplicateFiles, false);+ } else {+ toast.info(+ `All ${duplicateCount} selected file(s) already exist. Nothing was synced.`,+ );+ }+ }+ setPendingSync(null);}- }, [- selectedFiles,- connectorType,- syncMutation,- onOpenChange,- onIngestSuccess,- showShared,- ingestSettings,- ]);+ setDuplicateDialogOpen(open);+ };return (- <Dialog open={open} onOpenChange={onOpenChange}>- <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">- <DialogHeader>- <DialogTitle>Browse Files</DialogTitle>- <DialogDescription>- Select files to ingest from your {connectorType.replace("_", " ")}{" "}- connection.- {data && (- <span className="ml-1">- {data.total_ingested} of {data.total_remote} file(s) already- ingested.- </span>+ <>+ <Dialog open={open} onOpenChange={onOpenChange}>+ <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">+ <DialogHeader>+ <DialogTitle>Browse Files</DialogTitle>+ <DialogDescription>+ Select files to ingest from your {connectorType.replace("_", " ")}{" "}+ connection.+ {data && (+ <span className="ml-1">+ {data.total_ingested} of {data.total_remote} file(s) already+ ingested.+ </span>+ )}+ </DialogDescription>+ </DialogHeader>++ <div className="flex gap-2 items-center">+ {buckets && buckets.length > 1 && (+ <select+ className="border rounded px-2 py-1.5 text-sm bg-background"+ value={selectedBucket || ""}+ onChange={(e) => {+ setSelectedBucket(e.target.value || undefined);+ setSelectedFileIds(new Set());+ }}+ >+ {buckets.map((b) => (+ <option key={b} value={b}>+ {b}+ </option>+ ))}+ </select>)}- </DialogDescription>- </DialogHeader>-- <div className="flex gap-2 items-center">- {buckets && buckets.length > 1 && (- <select- className="border rounded px-2 py-1.5 text-sm bg-background"- value={selectedBucket || ""}- onChange={(e) => {- setSelectedBucket(e.target.value || undefined);- setSelectedFileIds(new Set());- }}- >- {buckets.map((b) => (- <option key={b} value={b}>- {b}- </option>- ))}- </select>- )}- <div className="relative flex-1">- <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />- <Input- placeholder="Search files..."- value={search}- onChange={(e) => setSearch(e.target.value)}- className="pl-9"- />+ <div className="relative flex-1">+ <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />+ <Input+ placeholder="Search files..."+ value={search}+ onChange={(e) => setSearch(e.target.value)}+ className="pl-9"+ />+ </div></div>- </div>- <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">- {isLoading ? (- <div className="flex items-center justify-center p-8">- <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />- <span className="ml-2 text-muted-foreground">- Loading files...- </span>- </div>- ) : error ? (- <div className="p-4 text-destructive text-sm">- Failed to load files:{" "}- {error instanceof Error ? error.message : "Unknown error"}- </div>- ) : files.length === 0 ? (- <div className="p-8 text-center text-muted-foreground text-sm">- {search.trim() && allFiles.length > 0- ? "No files match your search."- : "No files found."}- </div>- ) : (- <div className="divide-y">- {visibleSelectable.length > 0 && (- <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">- <input- type="checkbox"- checked={allVisibleSelected}- onChange={toggleAll}- className="h-4 w-4 rounded border border-input"- />- <span className="text-xs text-muted-foreground">- {selectedFileIds.size > 0- ? `${selectedFileIds.size} selected`- : `Select all (${visibleSelectable.length})`}- </span>- </div>- )}- {files.map((file) => (- <FileRow- key={file.id}- file={file}- selected={selectedFileIds.has(file.id)}- onToggle={() => toggleFile(file.id)}- />- ))}- </div>- )}- </ScrollArea>-- <DialogFooter>- <Button variant="outline" onClick={() => onOpenChange(false)}>- Cancel- </Button>- <Button- onClick={handleIngest}- disabled={selectedFiles.length === 0 || syncMutation.isPending}- >- {syncMutation.isPending ? (- <>- <Loader2 className="h-4 w-4 mr-2 animate-spin" />- Ingesting...- </>+ <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">+ {isLoading ? (+ <div className="flex items-center justify-center p-8">+ <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />+ <span className="ml-2 text-muted-foreground">+ Loading files...+ </span>+ </div>+ ) : error ? (+ <div className="p-4 text-destructive text-sm">+ Failed to load files:{" "}+ {error instanceof Error ? error.message : "Unknown error"}+ </div>+ ) : files.length === 0 ? (+ <div className="p-8 text-center text-muted-foreground text-sm">+ {search.trim() && allFiles.length > 0+ ? "No files match your search."+ : "No files found."}+ </div>) : (- <>- <Download className="h-4 w-4 mr-2" />- Ingest{" "}- {selectedFiles.length > 0- ? `${selectedFiles.length} file(s)`- : "selected"}- </>+ <div className="divide-y">+ {visibleSelectable.length > 0 && (+ <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">+ <input+ type="checkbox"+ checked={allVisibleSelected}+ onChange={toggleAll}+ className="h-4 w-4 rounded border border-input"+ />+ <span className="text-xs text-muted-foreground">+ {selectedFileIds.size > 0+ ? `${selectedFileIds.size} selected`+ : `Select all (${visibleSelectable.length})`}+ </span>+ </div>+ )}+ {files.map((file) => (+ <FileRow+ key={file.id}+ file={file}+ selected={selectedFileIds.has(file.id)}+ onToggle={() => toggleFile(file.id)}+ />+ ))}+ </div>)}- </Button>- </DialogFooter>- </DialogContent>- </Dialog>+ </ScrollArea>++ <DialogFooter>+ <Button variant="outline" onClick={() => onOpenChange(false)}>+ Cancel+ </Button>+ <Button+ onClick={handleIngest}+ disabled={+ selectedFiles.length === 0 ||+ syncMutation.isPending ||+ isCheckingDuplicates+ }+ >+ {syncMutation.isPending ? (+ <>+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />+ Ingesting...+ </>+ ) : isCheckingDuplicates ? (+ <>+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />+ Checking...+ </>+ ) : (+ <>+ <Download className="h-4 w-4 mr-2" />+ Ingest{" "}+ {selectedFiles.length > 0+ ? `${selectedFiles.length} file(s)`+ : "selected"}+ </>+ )}+ </Button>+ </DialogFooter>+ </DialogContent>+ </Dialog>++ <DuplicateHandlingDialog+ open={duplicateDialogOpen}+ onOpenChange={handleDuplicateDialogOpenChange}+ onOverwrite={handleOverwriteDuplicates}+ isLoading={syncMutation.isPending}+ duplicateNames={pendingSync?.duplicateNames}+ duplicateCount={pendingSync?.duplicateCount}+ />+ </>);}@@ -315,20 +441,11 @@ function FileRow({return idx >= 0 ? key.slice(0, idx + 1) : "";}, [file.key]);- // Unchanged ingested files are locked; stale ones (newer version at source) stay- // selectable so the user can re-ingest the update.- const locked = file.is_ingested && !file.is_stale;-return (- <label- className={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${- locked ? "opacity-60" : ""- }`}- >+ <label className="flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors"><inputtype="checkbox"- checked={selected || locked}- disabled={locked}+ checked={selected}onChange={onToggle}className="h-4 w-4 rounded border border-input"/>diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 7251537a3..c69e8fbe2 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -76,6 +76,15 @@ def _connector_sync_should_replace(connector_type: str) -> bool:return connector_type in ["google_drive", "sharepoint", "onedrive"]+def _is_unmapped_keyword_agg_error(err: Exception) -> bool:+ """True when a terms aggregation failed because the target field is an+ analyzed `text` field without fielddata enabled — the error OpenSearch+ raises for connector_file_id on indices that predate its addition to the+ explicit `keyword` mapping in config/settings.py."""+ msg = str(err)+ return "Text fields are not optimised" in msg or "fielddata" in msg++async def get_synced_file_ids_for_connector(connector_type: str,user_id: str,@@ -100,7 +109,7 @@ async def get_synced_file_ids_for_connector(try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- query_body = {+ query_body: dict[str, Any] = {"size": 0,"query": {"term": {"connector_type": connector_type}},"aggs": {@@ -116,7 +125,19 @@ async def get_synced_file_ids_for_connector(},}- result = await opensearch_client.search(index=get_index_name(), body=query_body)+ try:+ result = await opensearch_client.search(index=get_index_name(), body=query_body)+ except Exception as agg_err:+ if not _is_unmapped_keyword_agg_error(agg_err):+ raise+ # Some indices predate connector_file_id's addition to the explicit+ # mapping (config/settings.py), so it was dynamically mapped as+ # analyzed text instead of keyword — terms aggs need the+ # `.keyword` multi-field on those indices.+ query_body["aggs"]["unique_connector_file_ids"]["terms"]["field"] = (+ "connector_file_id.keyword"+ )+ result = await opensearch_client.search(index=get_index_name(), body=query_body)# Prefer connector_file_id — these are set by ConnectorFileProcessor (non-Langflow)# and hold the actual connector source IDs (e.g. SharePoint GUIDs), not SHA hashes.@@ -190,7 +211,7 @@ async def get_synced_id_to_filename_map(try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- query_body = {+ query_body: dict[str, Any] = {"size": 0,"query": {"term": {"connector_type": connector_type}},"aggs": {@@ -253,7 +274,7 @@ async def get_synced_id_to_modified_time_map(try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- query_body = {+ query_body: dict[str, Any] = {"size": 0,"query": {"term": {"connector_type": connector_type}},"aggs": {@@ -268,7 +289,17 @@ async def get_synced_id_to_modified_time_map(},}- result = await opensearch_client.search(index=get_index_name(), body=query_body)+ try:+ result = await opensearch_client.search(index=get_index_name(), body=query_body)+ except Exception as agg_err:+ if not _is_unmapped_keyword_agg_error(agg_err):+ raise+ # See get_synced_file_ids_for_connector: some indices predate the+ # explicit keyword mapping for connector_file_id.+ query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (+ "connector_file_id.keyword"+ )+ result = await opensearch_client.search(index=get_index_name(), body=query_body)aggs = result.get("aggregations", {})mapping: dict[str, float | None] = {}@@ -604,6 +635,11 @@ class ConnectorSyncBody(BaseModel):class ConnectorCheckDuplicatesBody(BaseModel):connection_id: str | None = Noneselected_files: list[Any] | None = None+ # Bucket-kind connectors (aws_s3, azure_blob, ibm_cos) select whole+ # buckets rather than individual files; when set (and selected_files is+ # not), the check lists files from these buckets and classifies each as+ # new/changed/unchanged instead of a plain filename match.+ bucket_filter: list[str] | None = Nonedef _connector_file_response(file_info: dict[str, Any], cleaned_name: str | None = None) -> dict:@@ -738,6 +774,83 @@ async def _classify_connector_duplicates(}+async def _classify_bucket_connector_duplicates(+ connector,+ connector_type: str,+ bucket_filter: list[str],+ session_manager,+ user_id: str,+ jwt_token: str | None,+) -> dict[str, Any]:+ """Preview a bucket_filter sync: classify remote blobs new/changed/unchanged+ without ingesting anything, mirroring the reconciliation in connector_sync's+ bucket_filter branch. "changed" blobs are reported as duplicates (they would+ overwrite an already-indexed version); "unchanged" blobs are silently+ dropped (the real sync would skip them too); "new" blobs are returned as+ ``non_duplicate_files`` so the caller can sync just those when the user+ chooses to skip duplicates.+ """+ original_buckets = connector.bucket_names+ connector.bucket_names = bucket_filter+ try:+ all_files: list[dict[str, Any]] = []+ page_token = None+ while True:+ result = await connector.list_files(page_token=page_token)+ all_files.extend(result.get("files", []))+ page_token = result.get("next_page_token")+ if not page_token:+ break+ finally:+ connector.bucket_names = original_buckets++ if not all_files:+ return {+ "duplicate_names": [],+ "duplicate_files": [],+ "non_duplicate_files": [],+ "duplicate_count": 0,+ "total_files": 0,+ }++ existing_ids, _, _ = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )+ existing_set = set(existing_ids)++ # Existence-based, like the OAuth connector duplicate check: any blob+ # already ingested under this connector_type is a "duplicate" regardless+ # of whether the remote copy is newer. (The real bucket_filter sync uses+ # modified_time to auto-skip unchanged blobs on ITS OWN — that's a+ # separate, silent optimization; the confirm dialog here is about whether+ # the user wants to touch an already-indexed file at all, same as it+ # would for Google Drive/OneDrive/SharePoint.)+ duplicate_files: list[dict[str, Any]] = []+ duplicate_names: list[str] = []+ non_duplicate_files: list[dict[str, Any]] = []+ for f in all_files:+ fid = f.get("id")+ if not fid:+ continue+ if fid in existing_set:+ response_file = _connector_file_response(f)+ duplicate_files.append(response_file)+ duplicate_names.append(response_file["name"])+ else:+ non_duplicate_files.append(_connector_file_response(f))++ return {+ "duplicate_names": list(dict.fromkeys(duplicate_names)),+ "duplicate_files": duplicate_files,+ "non_duplicate_files": non_duplicate_files,+ "duplicate_count": len(duplicate_files),+ "total_files": len(all_files),+ }++async def connector_check_duplicates(connector_type: str,body: ConnectorCheckDuplicatesBody,@@ -752,7 +865,7 @@ async def connector_check_duplicates(return deniedselected_files_raw = body.selected_files- if not selected_files_raw:+ if not selected_files_raw and not body.bucket_filter:return JSONResponse({"duplicate_names": []})try:@@ -794,6 +907,18 @@ async def connector_check_duplicates(status_code=404,)+ if body.bucket_filter and not selected_files_raw:+ return JSONResponse(+ await _classify_bucket_connector_duplicates(+ connector=connector,+ connector_type=connector_type,+ bucket_filter=body.bucket_filter,+ session_manager=session_manager,+ user_id=user.user_id,+ jwt_token=jwt_token,+ )+ )+return JSONResponse(await _classify_connector_duplicates(connector=connector,diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.pyindex 02c46e811..50ea274c9 100644--- a/tests/unit/api/test_connector_change_detection.py+++ b/tests/unit/api/test_connector_change_detection.py@@ -146,6 +146,56 @@ async def test_modified_time_map_prefers_connector_file_id_over_document_id(monkassert result["c::lf"] == 1704153600000.0+async def test_modified_time_map_falls_back_to_keyword_subfield_on_text_field_error(+ monkeypatch,+):+ """Indices that predate connector_file_id's addition to the explicit+ mapping (config/settings.py) have it dynamically mapped as analyzed text,+ and OpenSearch rejects a terms agg on that field outright. The helper+ must retry the aggregation against connector_file_id.keyword instead of+ treating this as a fatal error (which would make every blob look "new")."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ called_fields = []++ async def fake_search(*, index, body):+ called_fields.append(body["aggs"]["by_connector_file_id"]["terms"]["field"])+ if len(called_fields) == 1:+ raise Exception(+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "+ "are not optimised for operations that require per-document field "+ "data like aggregations and sorting, so these operations are "+ "disabled by default. Please use a keyword field instead. "+ "Alternatively, set fielddata=true on [connector_file_id]...')"+ )+ return {+ "aggregations": {+ "by_connector_file_id": {+ "buckets": [{"key": "c::a", "latest_modified": {"value": 1704067200000.0}}]+ },+ "by_document_id": {"buckets": []},+ }+ }++ opensearch_client.search = AsyncMock(side_effect=fake_search)+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_modified_time_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ assert result == {"c::a": 1704067200000.0}+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]++@pytest.mark.asyncioasync def test_modified_time_map_returns_empty_on_error(monkeypatch):from api import connectors as connectors_api@@ -165,6 +215,57 @@ async def test_modified_time_map_returns_empty_on_error(monkeypatch):assert result == {}+# ---------------------------------------------------------------------------+# get_synced_file_ids_for_connector+# ---------------------------------------------------------------------------+++async def test_synced_file_ids_falls_back_to_keyword_subfield_on_text_field_error(+ monkeypatch,+):+ """Same text-field mapping-drift issue as get_synced_id_to_modified_time_map,+ but here a fatal (uncaught) error would make orphan detection and bucket+ reconciliation treat every connector file as never-synced."""+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ called_fields = []++ async def fake_search(*, index, body):+ called_fields.append(body["aggs"]["unique_connector_file_ids"]["terms"]["field"])+ if len(called_fields) == 1:+ raise Exception(+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "+ "are not optimised for operations that require per-document field "+ "data...set fielddata=true on [connector_file_id]...')"+ )+ return {+ "aggregations": {+ "unique_connector_file_ids": {"buckets": [{"key": "c::a"}]},+ "unique_document_ids": {"buckets": []},+ "unique_filenames": {"buckets": []},+ }+ }++ opensearch_client.search = AsyncMock(side_effect=fake_search)+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ file_ids, filenames, id_field = await connectors_api.get_synced_file_ids_for_connector(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ assert file_ids == ["c::a"]+ assert id_field == "connector_file_id"+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]++# ---------------------------------------------------------------------------# connector_sync — bucket_filter reconciliation# ---------------------------------------------------------------------------diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.pyindex 388b64e3a..8d0d98246 100644--- a/tests/unit/connectors/test_connector_file_type_validation.py+++ b/tests/unit/connectors/test_connector_file_type_validation.py@@ -171,6 +171,98 @@ async def test_connector_check_duplicates():]+async def test_connector_check_duplicates_bucket_filter_is_existence_based():+ """Bucket-kind connectors (Azure Blob/S3/COS) select whole buckets, not+ individual files, so check-duplicates must accept bucket_filter and list+ files from those buckets. Classification must be existence-based — any+ blob already ingested under this connector_type is a duplicate — the same+ semantics the filename-based OAuth check uses, NOT the modified-time+ "changed" gate the real bucket_filter sync uses to silently skip unchanged+ blobs. Otherwise re-selecting an up-to-date bucket reports zero duplicates+ and the overwrite dialog never appears."""+ import json++ from fastapi.responses import JSONResponse++ from api.connectors import ConnectorCheckDuplicatesBody, connector_check_duplicates++ connector_service = MagicMock()+ connection_manager = MagicMock()+ connector_service.connection_manager = connection_manager++ connection = MagicMock()+ connection.connection_id = "conn-id"+ connection.is_active = True+ connection_manager.list_connections = AsyncMock(return_value=[connection])++ connector = MagicMock()+ connector.is_authenticated = True+ connector.authenticate = AsyncMock(return_value=True)+ connector.bucket_names = None+ connector.list_files = AsyncMock(+ return_value={+ "files": [+ # Already ingested (remote timestamp is irrelevant) -> duplicate.+ {+ "id": "bucket::already-ingested.pdf",+ "name": "already-ingested.pdf",+ "mimeType": "application/pdf",+ "modified_time": "2026-01-01T00:00:00Z",+ },+ # Not ingested yet -> new.+ {+ "id": "bucket::new.pdf",+ "name": "new.pdf",+ "mimeType": "application/pdf",+ "modified_time": "2026-07-13T00:00:00Z",+ },+ ]+ }+ )+ connector_service.get_connector = AsyncMock(return_value=connector)++ session_manager = MagicMock()+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={+ "aggregations": {+ "unique_connector_file_ids": {+ "buckets": [{"key": "bucket::already-ingested.pdf"}],+ },+ "unique_document_ids": {"buckets": []},+ "unique_filenames": {"buckets": []},+ }+ }+ )+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ user = MagicMock()+ user.user_id = "user-id"+ user.jwt_token = "jwt-token"++ body = ConnectorCheckDuplicatesBody(+ connection_id="conn-id",+ bucket_filter=["bucket"],+ )++ response = await connector_check_duplicates(+ connector_type="azure_blob",+ body=body,+ request=MagicMock(),+ connector_service=connector_service,+ session_manager=session_manager,+ user=user,+ )++ assert isinstance(response, JSONResponse)+ data = json.loads(response.body.decode())+ assert data["duplicate_names"] == ["already-ingested.pdf"]+ assert data["duplicate_count"] == 1+ assert data["total_files"] == 2+ assert [f["id"] for f in data["non_duplicate_files"]] == ["bucket::new.pdf"]++@pytest.mark.asyncioasync def test_connector_sync_skip_duplicates_returns_no_files_when_all_selected_are_duplicates(monkeypatch,