Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: user-facing duplicate handling workflow for bucket-based connectors (backport of #2077) (#2096)

Edwin Josefe9425d02294
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsx
index c4a6800c6..a549c2aa3 100644
--- a/frontend/components/connectors/shared-bucket-view.tsx
+++ b/frontend/components/connectors/shared-bucket-view.tsx
@@ -8,12 +8,28 @@ import type { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";
import { IngestSettings } from "@/components/cloud-picker/ingest-settings";
import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";
+import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog";
import { FileBrowserDialog } from "@/components/file-browser-dialog";
import { Button } from "@/components/ui/button";
import { useAuth } from "@/contexts/auth-context";
import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";
+interface BucketDuplicateFile {
+ id: string;
+ name: string;
+ mimeType: string;
+ downloadUrl?: string;
+ size?: number;
+}
+
+interface BucketDuplicateCheckResponse {
+ duplicate_names?: string[];
+ duplicate_count?: number;
+ duplicate_files?: BucketDuplicateFile[];
+ non_duplicate_files?: BucketDuplicateFile[];
+}
+
export interface SharedBucketViewProps {
connector: any;
buckets: Array<{ name: string; ingested_count: number }> | undefined;
@@ -70,6 +86,15 @@ export function SharedBucketView({
const [browseDialogBucket, setBrowseDialogBucket] = useState<string | null>(
null,
);
+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);
+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);
+ const [pendingDuplicates, setPendingDuplicates] = useState<{
+ duplicateNames: string[];
+ duplicateCount: number;
+ duplicateFiles: BucketDuplicateFile[];
+ nonDuplicateFiles: BucketDuplicateFile[];
+ } | null>(null);
+ const isOverwriteConfirmedRef = useRef(false);
useEffect(() => {
if (
@@ -111,20 +136,43 @@ export function SharedBucketView({
});
};
- const ingestSelected = () => {
- const chunkErr = getIngestChunkSettingsError(ingestSettings);
- if (chunkErr) {
- toast.error("Could not start ingest", { description: chunkErr });
- return;
- }
- trackStartProcess({
+ const onSyncSettled = () => {
+ invalidate();
+ };
+
+ const onSyncError = (err: unknown) => {
+ trackProcessFailure({
processType: "Ingestion",
process: "Document Upload",
category: "Knowledge",
source: "connector",
connector_type: connector.type,
- total_buckets: selectedBuckets.size,
+ resultValue: err instanceof Error ? err.message : "Sync failed",
});
+ toast.error(err instanceof Error ? err.message : "Sync failed");
+ };
+
+ const onSyncSuccess = (result: { task_ids?: string[]; message?: string }) => {
+ onSyncSettled();
+ if (result.task_ids?.length) {
+ // The container path may return two tasks (new files + changed files);
+ // track them all.
+ for (const id of result.task_ids) {
+ addTask(id, {
+ connectorType: connector.type,
+ source: "connector",
+ });
+ }
+ onDone();
+ } else {
+ toast.info(result.message ?? "No files found in the selected buckets.");
+ }
+ };
+
+ // Full bucket sync: backend classifies new/changed/unchanged itself and
+ // re-ingests both new and changed blobs (used both when there are no
+ // duplicates and when the user chooses to overwrite them).
+ const runBucketSync = () => {
syncMutation.mutate(
{
connectorType: connector.type,
@@ -138,35 +186,129 @@ export function SharedBucketView({
: undefined,
},
},
+ { onSuccess: onSyncSuccess, onError: onSyncError },
+ );
+ };
+
+ // Sync explicit files. `replaceDuplicates` forces an unconditional
+ // re-ingest (bypassing the bucket_filter path's modified-time gate) —
+ // used when the user confirms "Overwrite duplicates" so an already
+ // up-to-date file still gets re-ingested, matching what "overwrite" means
+ // for the OAuth connectors.
+ const runSelectedFilesSync = (
+ files: BucketDuplicateFile[],
+ replaceDuplicates = false,
+ ) => {
+ syncMutation.mutate(
{
- onSuccess: (result) => {
- invalidate();
- if (result.task_ids?.length) {
- // The container path may return two tasks (new files + changed files);
- // track them all.
- trackIngestTasks(result.task_ids);
- onDone();
- } else {
- toast.info(
- result.message ?? "No files found in the selected buckets.",
- );
- }
- },
- onError: (err) => {
- trackProcessFailure({
- processType: "Ingestion",
- process: "Document Upload",
- category: "Knowledge",
- source: "connector",
- connector_type: connector.type,
- resultValue: err instanceof Error ? err.message : "Sync failed",
- });
- toast.error(err instanceof Error ? err.message : "Sync failed");
+ connectorType: connector.type,
+ body: {
+ connection_id: connector.connectionId!,
+ selected_files: files,
+ settings: ingestSettings,
+ replace_duplicates: replaceDuplicates,
+ shared: showSharedToggle
+ ? (ingestSettings.shared ?? false)
+ : undefined,
},
},
+ { onSuccess: onSyncSuccess, onError: onSyncError },
);
};
+ const ingestSelected = async () => {
+ const chunkErr = getIngestChunkSettingsError(ingestSettings);
+ if (chunkErr) {
+ toast.error("Could not start ingest", { description: chunkErr });
+ return;
+ }
+ trackStartProcess({
+ processType: "Ingestion",
+ process: "Document Upload",
+ category: "Knowledge",
+ source: "connector",
+ connector_type: connector.type,
+ total_buckets: selectedBuckets.size,
+ });
+
+ setIsCheckingDuplicates(true);
+ try {
+ const checkResponse = await fetch(
+ `/api/connectors/${connector.type}/check-duplicates`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ connection_id: connector.connectionId,
+ bucket_filter: Array.from(selectedBuckets),
+ }),
+ },
+ );
+
+ if (!checkResponse.ok) {
+ throw new Error(`Duplicate check failed: ${checkResponse.statusText}`);
+ }
+
+ const checkData =
+ (await checkResponse.json()) as BucketDuplicateCheckResponse;
+ const duplicateNames = checkData.duplicate_names || [];
+ const duplicateCount =
+ typeof checkData.duplicate_count === "number"
+ ? checkData.duplicate_count
+ : duplicateNames.length;
+
+ if (duplicateCount === 0) {
+ runBucketSync();
+ return;
+ }
+
+ setPendingDuplicates({
+ duplicateNames,
+ duplicateCount,
+ duplicateFiles: checkData.duplicate_files || [],
+ nonDuplicateFiles: checkData.non_duplicate_files || [],
+ });
+ setDuplicateDialogOpen(true);
+ } catch (err) {
+ console.error("[Bucket Sync] Duplicate check failed:", err);
+ // Fallback: proceed with the normal full sync (backend still handles
+ // new/changed reconciliation on its own).
+ runBucketSync();
+ } finally {
+ setIsCheckingDuplicates(false);
+ }
+ };
+
+ const handleOverwriteDuplicates = () => {
+ if (!pendingDuplicates) return;
+ isOverwriteConfirmedRef.current = true;
+ const { duplicateFiles, nonDuplicateFiles } = pendingDuplicates;
+ runSelectedFilesSync([...duplicateFiles, ...nonDuplicateFiles], true);
+ setPendingDuplicates(null);
+ };
+
+ const handleDuplicateDialogOpenChange = (open: boolean) => {
+ if (!open && pendingDuplicates) {
+ if (isOverwriteConfirmedRef.current) {
+ // Overwrite already submitted in handleOverwriteDuplicates; this close
+ // event fires immediately after and would otherwise re-enter the
+ // "skip duplicates" branch.
+ isOverwriteConfirmedRef.current = false;
+ } else {
+ const { nonDuplicateFiles, duplicateCount } = pendingDuplicates;
+ if (nonDuplicateFiles.length > 0) {
+ runSelectedFilesSync(nonDuplicateFiles);
+ } else {
+ toast.info(
+ `All ${duplicateCount} file(s) already exist and were skipped. Nothing was synced.`,
+ );
+ }
+ }
+ setPendingDuplicates(null);
+ }
+ setDuplicateDialogOpen(open);
+ };
+
return (
<>
<div className="mb-8 flex gap-2 items-center">
@@ -330,14 +472,20 @@ export function SharedBucketView({
<Button
className="bg-foreground text-background hover:bg-foreground/90 font-semibold"
onClick={ingestSelected}
- disabled={syncMutation.isPending || selectedBuckets.size === 0}
- loading={syncMutation.isPending}
+ disabled={
+ syncMutation.isPending ||
+ isCheckingDuplicates ||
+ selectedBuckets.size === 0
+ }
+ loading={syncMutation.isPending || isCheckingDuplicates}
>
{syncMutation.isPending
? "Ingesting…"
- : selectedBuckets.size > 0
- ? `Ingest ${selectedBuckets.size} Bucket${selectedBuckets.size !== 1 ? "s" : ""}`
- : "Select Buckets to Ingest"}
+ : isCheckingDuplicates
+ ? "Checking…"
+ : selectedBuckets.size > 0
+ ? `Ingest ${selectedBuckets.size} Bucket${selectedBuckets.size !== 1 ? "s" : ""}`
+ : "Select Buckets to Ingest"}
</Button>
</div>
</div>
@@ -366,6 +514,15 @@ export function SharedBucketView({
ingestSettings={ingestSettings}
/>
)}
+
+ <DuplicateHandlingDialog
+ open={duplicateDialogOpen}
+ onOpenChange={handleDuplicateDialogOpenChange}
+ onOverwrite={handleOverwriteDuplicates}
+ isLoading={syncMutation.isPending}
+ duplicateNames={pendingDuplicates?.duplicateNames}
+ duplicateCount={pendingDuplicates?.duplicateCount}
+ />
</>
);
}
diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsx
index 1e89424b5..84149eed9 100644
--- a/frontend/components/file-browser-dialog.tsx
+++ b/frontend/components/file-browser-dialog.tsx
@@ -1,7 +1,7 @@
"use client";
import { Check, Download, Loader2, RefreshCw, Search } from "lucide-react";
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
import {
@@ -10,6 +10,7 @@ import {
} from "@/app/api/queries/useBrowseConnectionFiles";
import type { IngestSettings } from "@/components/cloud-picker/types";
import { formatFileSize } from "@/lib/file-format";
+import { DuplicateHandlingDialog } from "./duplicate-handling-dialog";
import { Badge } from "./ui/badge";
import { Button } from "./ui/button";
import {
@@ -23,6 +24,19 @@ import {
import { Input } from "./ui/input";
import { ScrollArea } from "./ui/scroll-area";
+interface SyncFilePayload {
+ id: string;
+ name: string;
+ mimeType: string;
+ size?: number;
+}
+
+interface FileBrowserDuplicateCheckResponse {
+ duplicate_names?: string[];
+ duplicate_count?: number;
+ non_duplicate_files?: SyncFilePayload[];
+}
+
interface FileBrowserDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
@@ -80,12 +94,10 @@ export function FileBrowserDialog({
}, [allFiles, search]);
// Selectable files within the current (filtered) view — drives "Select all".
- // A file is selectable when it's not yet ingested, or it's ingested but stale
- // (a newer version exists at the source, so re-ingest is allowed).
- const visibleSelectable = useMemo(
- () => files.filter((f) => !f.is_ingested || f.is_stale),
- [files],
- );
+ // Already-ingested files are selectable too (re-checking one is an explicit
+ // request to re-ingest it — handleIngest routes through the duplicate-check
+ // confirmation flow before overwriting).
+ const visibleSelectable = files;
const allVisibleSelected =
visibleSelectable.length > 0 &&
@@ -127,171 +139,285 @@ export function FileBrowserDialog({
[allFiles, selectedFileIds],
);
+ const [isCheckingDuplicates, setIsCheckingDuplicates] = useState(false);
+ const [duplicateDialogOpen, setDuplicateDialogOpen] = useState(false);
+ const [pendingSync, setPendingSync] = useState<{
+ allFiles: SyncFilePayload[];
+ nonDuplicateFiles: SyncFilePayload[];
+ duplicateNames: string[];
+ duplicateCount: number;
+ } | null>(null);
+ const isOverwriteConfirmedRef = useRef(false);
+
+ const submitSync = useCallback(
+ async (syncFiles: SyncFilePayload[], replaceDuplicates: boolean) => {
+ try {
+ const result = await syncMutation.mutateAsync({
+ connectorType,
+ body: {
+ selected_files: syncFiles,
+ settings: ingestSettings,
+ shared: showShared ? (ingestSettings?.shared ?? false) : undefined,
+ replace_duplicates: replaceDuplicates,
+ },
+ });
+
+ toast.success("Ingestion started", {
+ description: `${syncFiles.length} file(s) queued for ingestion.`,
+ });
+
+ setSelectedFileIds(new Set());
+ onOpenChange(false);
+ onIngestSuccess?.(result);
+ } catch (err) {
+ toast.error("Ingestion failed", {
+ description: err instanceof Error ? err.message : "Unknown error",
+ });
+ }
+ },
+ [
+ connectorType,
+ syncMutation,
+ onOpenChange,
+ onIngestSuccess,
+ showShared,
+ ingestSettings,
+ ],
+ );
+
const handleIngest = useCallback(async () => {
if (selectedFiles.length === 0) return;
- // If any selected file is a stale re-ingest, replace the indexed copy so the
- // newer version overwrites the old chunks (rather than being skipped as a
- // duplicate filename). New files are unaffected by this flag.
+ const filesPayload: SyncFilePayload[] = selectedFiles.map((f) => ({
+ id: f.id,
+ name: f.name,
+ mimeType: "",
+ size: f.size,
+ }));
+
+ // If any selected file is a stale re-ingest, default to replacing the
+ // indexed copy so the newer version overwrites the old chunks even if
+ // the duplicate check below is skipped (network error) or doesn't flag
+ // it. New files are unaffected by this flag.
const hasStale = selectedFiles.some((f) => f.is_stale);
+ setIsCheckingDuplicates(true);
try {
- const result = await syncMutation.mutateAsync({
- connectorType,
- body: {
- selected_files: selectedFiles.map((f) => ({
- id: f.id,
- name: f.name,
- mimeType: "",
- size: f.size,
- })),
- settings: ingestSettings,
- shared: showShared ? (ingestSettings?.shared ?? false) : undefined,
- ...(hasStale ? { replace_duplicates: true } : {}),
+ const checkResponse = await fetch(
+ `/api/connectors/${connectorType}/check-duplicates`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ connection_id: connectionId,
+ selected_files: filesPayload,
+ }),
},
- });
+ );
- toast.success("Ingestion started", {
- description: `${selectedFiles.length} file(s) queued for ingestion.`,
- });
+ if (!checkResponse.ok) {
+ throw new Error(`Duplicate check failed: ${checkResponse.statusText}`);
+ }
- setSelectedFileIds(new Set());
- onOpenChange(false);
- onIngestSuccess?.(result);
- } catch (err) {
- toast.error("Ingestion failed", {
- description: err instanceof Error ? err.message : "Unknown error",
+ const checkData =
+ (await checkResponse.json()) as FileBrowserDuplicateCheckResponse;
+ const duplicateNames = checkData.duplicate_names || [];
+ const duplicateCount =
+ typeof checkData.duplicate_count === "number"
+ ? checkData.duplicate_count
+ : duplicateNames.length;
+
+ if (duplicateCount === 0) {
+ await submitSync(filesPayload, hasStale);
+ return;
+ }
+
+ setPendingSync({
+ allFiles: filesPayload,
+ nonDuplicateFiles: checkData.non_duplicate_files || [],
+ duplicateNames,
+ duplicateCount,
});
+ setDuplicateDialogOpen(true);
+ } catch (err) {
+ console.error("[File Browser] Duplicate check failed:", err);
+ // Fallback: proceed without overwrite (backend will still skip
+ // exact-duplicate filenames on its own), unless we already know some
+ // selected files are stale re-ingests.
+ await submitSync(filesPayload, hasStale);
+ } finally {
+ setIsCheckingDuplicates(false);
+ }
+ }, [selectedFiles, connectorType, connectionId, submitSync]);
+
+ const handleOverwriteDuplicates = () => {
+ if (!pendingSync) return;
+ isOverwriteConfirmedRef.current = true;
+ submitSync(pendingSync.allFiles, true);
+ setPendingSync(null);
+ };
+
+ const handleDuplicateDialogOpenChange = (open: boolean) => {
+ if (!open && pendingSync) {
+ if (isOverwriteConfirmedRef.current) {
+ // Overwrite already submitted in handleOverwriteDuplicates; this close
+ // event fires immediately after and would otherwise re-enter the
+ // "skip duplicates" branch.
+ isOverwriteConfirmedRef.current = false;
+ } else {
+ const { nonDuplicateFiles, duplicateCount } = pendingSync;
+ if (nonDuplicateFiles.length > 0) {
+ submitSync(nonDuplicateFiles, false);
+ } else {
+ toast.info(
+ `All ${duplicateCount} selected file(s) already exist. Nothing was synced.`,
+ );
+ }
+ }
+ setPendingSync(null);
}
- }, [
- selectedFiles,
- connectorType,
- syncMutation,
- onOpenChange,
- onIngestSuccess,
- showShared,
- ingestSettings,
- ]);
+ setDuplicateDialogOpen(open);
+ };
return (
- <Dialog open={open} onOpenChange={onOpenChange}>
- <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">
- <DialogHeader>
- <DialogTitle>Browse Files</DialogTitle>
- <DialogDescription>
- Select files to ingest from your {connectorType.replace("_", " ")}{" "}
- connection.
- {data && (
- <span className="ml-1">
- {data.total_ingested} of {data.total_remote} file(s) already
- ingested.
- </span>
+ <>
+ <Dialog open={open} onOpenChange={onOpenChange}>
+ <DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">
+ <DialogHeader>
+ <DialogTitle>Browse Files</DialogTitle>
+ <DialogDescription>
+ Select files to ingest from your {connectorType.replace("_", " ")}{" "}
+ connection.
+ {data && (
+ <span className="ml-1">
+ {data.total_ingested} of {data.total_remote} file(s) already
+ ingested.
+ </span>
+ )}
+ </DialogDescription>
+ </DialogHeader>
+
+ <div className="flex gap-2 items-center">
+ {buckets && buckets.length > 1 && (
+ <select
+ className="border rounded px-2 py-1.5 text-sm bg-background"
+ value={selectedBucket || ""}
+ onChange={(e) => {
+ setSelectedBucket(e.target.value || undefined);
+ setSelectedFileIds(new Set());
+ }}
+ >
+ {buckets.map((b) => (
+ <option key={b} value={b}>
+ {b}
+ </option>
+ ))}
+ </select>
)}
- </DialogDescription>
- </DialogHeader>
-
- <div className="flex gap-2 items-center">
- {buckets && buckets.length > 1 && (
- <select
- className="border rounded px-2 py-1.5 text-sm bg-background"
- value={selectedBucket || ""}
- onChange={(e) => {
- setSelectedBucket(e.target.value || undefined);
- setSelectedFileIds(new Set());
- }}
- >
- {buckets.map((b) => (
- <option key={b} value={b}>
- {b}
- </option>
- ))}
- </select>
- )}
- <div className="relative flex-1">
- <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
- <Input
- placeholder="Search files..."
- value={search}
- onChange={(e) => setSearch(e.target.value)}
- className="pl-9"
- />
+ <div className="relative flex-1">
+ <Search className="absolute left-2.5 top-2.5 h-4 w-4 text-muted-foreground" />
+ <Input
+ placeholder="Search files..."
+ value={search}
+ onChange={(e) => setSearch(e.target.value)}
+ className="pl-9"
+ />
+ </div>
</div>
- </div>
- <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">
- {isLoading ? (
- <div className="flex items-center justify-center p-8">
- <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
- <span className="ml-2 text-muted-foreground">
- Loading files...
- </span>
- </div>
- ) : error ? (
- <div className="p-4 text-destructive text-sm">
- Failed to load files:{" "}
- {error instanceof Error ? error.message : "Unknown error"}
- </div>
- ) : files.length === 0 ? (
- <div className="p-8 text-center text-muted-foreground text-sm">
- {search.trim() && allFiles.length > 0
- ? "No files match your search."
- : "No files found."}
- </div>
- ) : (
- <div className="divide-y">
- {visibleSelectable.length > 0 && (
- <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">
- <input
- type="checkbox"
- checked={allVisibleSelected}
- onChange={toggleAll}
- className="h-4 w-4 rounded border border-input"
- />
- <span className="text-xs text-muted-foreground">
- {selectedFileIds.size > 0
- ? `${selectedFileIds.size} selected`
- : `Select all (${visibleSelectable.length})`}
- </span>
- </div>
- )}
- {files.map((file) => (
- <FileRow
- key={file.id}
- file={file}
- selected={selectedFileIds.has(file.id)}
- onToggle={() => toggleFile(file.id)}
- />
- ))}
- </div>
- )}
- </ScrollArea>
-
- <DialogFooter>
- <Button variant="outline" onClick={() => onOpenChange(false)}>
- Cancel
- </Button>
- <Button
- onClick={handleIngest}
- disabled={selectedFiles.length === 0 || syncMutation.isPending}
- >
- {syncMutation.isPending ? (
- <>
- <Loader2 className="h-4 w-4 mr-2 animate-spin" />
- Ingesting...
- </>
+ <ScrollArea className="flex-1 min-h-0 max-h-[400px] border rounded">
+ {isLoading ? (
+ <div className="flex items-center justify-center p-8">
+ <Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
+ <span className="ml-2 text-muted-foreground">
+ Loading files...
+ </span>
+ </div>
+ ) : error ? (
+ <div className="p-4 text-destructive text-sm">
+ Failed to load files:{" "}
+ {error instanceof Error ? error.message : "Unknown error"}
+ </div>
+ ) : files.length === 0 ? (
+ <div className="p-8 text-center text-muted-foreground text-sm">
+ {search.trim() && allFiles.length > 0
+ ? "No files match your search."
+ : "No files found."}
+ </div>
) : (
- <>
- <Download className="h-4 w-4 mr-2" />
- Ingest{" "}
- {selectedFiles.length > 0
- ? `${selectedFiles.length} file(s)`
- : "selected"}
- </>
+ <div className="divide-y">
+ {visibleSelectable.length > 0 && (
+ <div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">
+ <input
+ type="checkbox"
+ checked={allVisibleSelected}
+ onChange={toggleAll}
+ className="h-4 w-4 rounded border border-input"
+ />
+ <span className="text-xs text-muted-foreground">
+ {selectedFileIds.size > 0
+ ? `${selectedFileIds.size} selected`
+ : `Select all (${visibleSelectable.length})`}
+ </span>
+ </div>
+ )}
+ {files.map((file) => (
+ <FileRow
+ key={file.id}
+ file={file}
+ selected={selectedFileIds.has(file.id)}
+ onToggle={() => toggleFile(file.id)}
+ />
+ ))}
+ </div>
)}
- </Button>
- </DialogFooter>
- </DialogContent>
- </Dialog>
+ </ScrollArea>
+
+ <DialogFooter>
+ <Button variant="outline" onClick={() => onOpenChange(false)}>
+ Cancel
+ </Button>
+ <Button
+ onClick={handleIngest}
+ disabled={
+ selectedFiles.length === 0 ||
+ syncMutation.isPending ||
+ isCheckingDuplicates
+ }
+ >
+ {syncMutation.isPending ? (
+ <>
+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+ Ingesting...
+ </>
+ ) : isCheckingDuplicates ? (
+ <>
+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+ Checking...
+ </>
+ ) : (
+ <>
+ <Download className="h-4 w-4 mr-2" />
+ Ingest{" "}
+ {selectedFiles.length > 0
+ ? `${selectedFiles.length} file(s)`
+ : "selected"}
+ </>
+ )}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+
+ <DuplicateHandlingDialog
+ open={duplicateDialogOpen}
+ onOpenChange={handleDuplicateDialogOpenChange}
+ onOverwrite={handleOverwriteDuplicates}
+ isLoading={syncMutation.isPending}
+ duplicateNames={pendingSync?.duplicateNames}
+ duplicateCount={pendingSync?.duplicateCount}
+ />
+ </>
);
}
@@ -315,20 +441,11 @@ function FileRow({
return idx >= 0 ? key.slice(0, idx + 1) : "";
}, [file.key]);
- // Unchanged ingested files are locked; stale ones (newer version at source) stay
- // selectable so the user can re-ingest the update.
- const locked = file.is_ingested && !file.is_stale;
-
return (
- <label
- className={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${
- locked ? "opacity-60" : ""
- }`}
- >
+ <label className="flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors">
<input
type="checkbox"
- checked={selected || locked}
- disabled={locked}
+ checked={selected}
onChange={onToggle}
className="h-4 w-4 rounded border border-input"
/>
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 7251537a3..c69e8fbe2 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -76,6 +76,15 @@ def _connector_sync_should_replace(connector_type: str) -> bool:
return connector_type in ["google_drive", "sharepoint", "onedrive"]
+def _is_unmapped_keyword_agg_error(err: Exception) -> bool:
+ """True when a terms aggregation failed because the target field is an
+ analyzed `text` field without fielddata enabled — the error OpenSearch
+ raises for connector_file_id on indices that predate its addition to the
+ explicit `keyword` mapping in config/settings.py."""
+ msg = str(err)
+ return "Text fields are not optimised" in msg or "fielddata" in msg
+
+
async def get_synced_file_ids_for_connector(
connector_type: str,
user_id: str,
@@ -100,7 +109,7 @@ async def get_synced_file_ids_for_connector(
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- query_body = {
+ query_body: dict[str, Any] = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
@@ -116,7 +125,19 @@ async def get_synced_file_ids_for_connector(
},
}
- result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ try:
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ except Exception as agg_err:
+ if not _is_unmapped_keyword_agg_error(agg_err):
+ raise
+ # Some indices predate connector_file_id's addition to the explicit
+ # mapping (config/settings.py), so it was dynamically mapped as
+ # analyzed text instead of keyword — terms aggs need the
+ # `.keyword` multi-field on those indices.
+ query_body["aggs"]["unique_connector_file_ids"]["terms"]["field"] = (
+ "connector_file_id.keyword"
+ )
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
# Prefer connector_file_id — these are set by ConnectorFileProcessor (non-Langflow)
# and hold the actual connector source IDs (e.g. SharePoint GUIDs), not SHA hashes.
@@ -190,7 +211,7 @@ async def get_synced_id_to_filename_map(
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- query_body = {
+ query_body: dict[str, Any] = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
@@ -253,7 +274,7 @@ async def get_synced_id_to_modified_time_map(
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- query_body = {
+ query_body: dict[str, Any] = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
@@ -268,7 +289,17 @@ async def get_synced_id_to_modified_time_map(
},
}
- result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ try:
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ except Exception as agg_err:
+ if not _is_unmapped_keyword_agg_error(agg_err):
+ raise
+ # See get_synced_file_ids_for_connector: some indices predate the
+ # explicit keyword mapping for connector_file_id.
+ query_body["aggs"]["by_connector_file_id"]["terms"]["field"] = (
+ "connector_file_id.keyword"
+ )
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
aggs = result.get("aggregations", {})
mapping: dict[str, float | None] = {}
@@ -604,6 +635,11 @@ class ConnectorSyncBody(BaseModel):
class ConnectorCheckDuplicatesBody(BaseModel):
connection_id: str | None = None
selected_files: list[Any] | None = None
+ # Bucket-kind connectors (aws_s3, azure_blob, ibm_cos) select whole
+ # buckets rather than individual files; when set (and selected_files is
+ # not), the check lists files from these buckets and classifies each as
+ # new/changed/unchanged instead of a plain filename match.
+ bucket_filter: list[str] | None = None
def _connector_file_response(file_info: dict[str, Any], cleaned_name: str | None = None) -> dict:
@@ -738,6 +774,83 @@ async def _classify_connector_duplicates(
}
+async def _classify_bucket_connector_duplicates(
+ connector,
+ connector_type: str,
+ bucket_filter: list[str],
+ session_manager,
+ user_id: str,
+ jwt_token: str | None,
+) -> dict[str, Any]:
+ """Preview a bucket_filter sync: classify remote blobs new/changed/unchanged
+ without ingesting anything, mirroring the reconciliation in connector_sync's
+ bucket_filter branch. "changed" blobs are reported as duplicates (they would
+ overwrite an already-indexed version); "unchanged" blobs are silently
+ dropped (the real sync would skip them too); "new" blobs are returned as
+ ``non_duplicate_files`` so the caller can sync just those when the user
+ chooses to skip duplicates.
+ """
+ original_buckets = connector.bucket_names
+ connector.bucket_names = bucket_filter
+ try:
+ all_files: list[dict[str, Any]] = []
+ page_token = None
+ while True:
+ result = await connector.list_files(page_token=page_token)
+ all_files.extend(result.get("files", []))
+ page_token = result.get("next_page_token")
+ if not page_token:
+ break
+ finally:
+ connector.bucket_names = original_buckets
+
+ if not all_files:
+ return {
+ "duplicate_names": [],
+ "duplicate_files": [],
+ "non_duplicate_files": [],
+ "duplicate_count": 0,
+ "total_files": 0,
+ }
+
+ existing_ids, _, _ = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+ existing_set = set(existing_ids)
+
+ # Existence-based, like the OAuth connector duplicate check: any blob
+ # already ingested under this connector_type is a "duplicate" regardless
+ # of whether the remote copy is newer. (The real bucket_filter sync uses
+ # modified_time to auto-skip unchanged blobs on ITS OWN — that's a
+ # separate, silent optimization; the confirm dialog here is about whether
+ # the user wants to touch an already-indexed file at all, same as it
+ # would for Google Drive/OneDrive/SharePoint.)
+ duplicate_files: list[dict[str, Any]] = []
+ duplicate_names: list[str] = []
+ non_duplicate_files: list[dict[str, Any]] = []
+ for f in all_files:
+ fid = f.get("id")
+ if not fid:
+ continue
+ if fid in existing_set:
+ response_file = _connector_file_response(f)
+ duplicate_files.append(response_file)
+ duplicate_names.append(response_file["name"])
+ else:
+ non_duplicate_files.append(_connector_file_response(f))
+
+ return {
+ "duplicate_names": list(dict.fromkeys(duplicate_names)),
+ "duplicate_files": duplicate_files,
+ "non_duplicate_files": non_duplicate_files,
+ "duplicate_count": len(duplicate_files),
+ "total_files": len(all_files),
+ }
+
+
async def connector_check_duplicates(
connector_type: str,
body: ConnectorCheckDuplicatesBody,
@@ -752,7 +865,7 @@ async def connector_check_duplicates(
return denied
selected_files_raw = body.selected_files
- if not selected_files_raw:
+ if not selected_files_raw and not body.bucket_filter:
return JSONResponse({"duplicate_names": []})
try:
@@ -794,6 +907,18 @@ async def connector_check_duplicates(
status_code=404,
)
+ if body.bucket_filter and not selected_files_raw:
+ return JSONResponse(
+ await _classify_bucket_connector_duplicates(
+ connector=connector,
+ connector_type=connector_type,
+ bucket_filter=body.bucket_filter,
+ session_manager=session_manager,
+ user_id=user.user_id,
+ jwt_token=jwt_token,
+ )
+ )
+
return JSONResponse(
await _classify_connector_duplicates(
connector=connector,
diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.py
index 02c46e811..50ea274c9 100644
--- a/tests/unit/api/test_connector_change_detection.py
+++ b/tests/unit/api/test_connector_change_detection.py
@@ -146,6 +146,56 @@ async def test_modified_time_map_prefers_connector_file_id_over_document_id(monk
assert result["c::lf"] == 1704153600000.0
+async def test_modified_time_map_falls_back_to_keyword_subfield_on_text_field_error(
+ monkeypatch,
+):
+ """Indices that predate connector_file_id's addition to the explicit
+ mapping (config/settings.py) have it dynamically mapped as analyzed text,
+ and OpenSearch rejects a terms agg on that field outright. The helper
+ must retry the aggregation against connector_file_id.keyword instead of
+ treating this as a fatal error (which would make every blob look "new")."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ called_fields = []
+
+ async def fake_search(*, index, body):
+ called_fields.append(body["aggs"]["by_connector_file_id"]["terms"]["field"])
+ if len(called_fields) == 1:
+ raise Exception(
+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "
+ "are not optimised for operations that require per-document field "
+ "data like aggregations and sorting, so these operations are "
+ "disabled by default. Please use a keyword field instead. "
+ "Alternatively, set fielddata=true on [connector_file_id]...')"
+ )
+ return {
+ "aggregations": {
+ "by_connector_file_id": {
+ "buckets": [{"key": "c::a", "latest_modified": {"value": 1704067200000.0}}]
+ },
+ "by_document_id": {"buckets": []},
+ }
+ }
+
+ opensearch_client.search = AsyncMock(side_effect=fake_search)
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_modified_time_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ assert result == {"c::a": 1704067200000.0}
+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]
+
+
@pytest.mark.asyncio
async def test_modified_time_map_returns_empty_on_error(monkeypatch):
from api import connectors as connectors_api
@@ -165,6 +215,57 @@ async def test_modified_time_map_returns_empty_on_error(monkeypatch):
assert result == {}
+# ---------------------------------------------------------------------------
+# get_synced_file_ids_for_connector
+# ---------------------------------------------------------------------------
+
+
+async def test_synced_file_ids_falls_back_to_keyword_subfield_on_text_field_error(
+ monkeypatch,
+):
+ """Same text-field mapping-drift issue as get_synced_id_to_modified_time_map,
+ but here a fatal (uncaught) error would make orphan detection and bucket
+ reconciliation treat every connector file as never-synced."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ called_fields = []
+
+ async def fake_search(*, index, body):
+ called_fields.append(body["aggs"]["unique_connector_file_ids"]["terms"]["field"])
+ if len(called_fields) == 1:
+ raise Exception(
+ "RequestError(400, 'search_phase_execution_exception', 'Text fields "
+ "are not optimised for operations that require per-document field "
+ "data...set fielddata=true on [connector_file_id]...')"
+ )
+ return {
+ "aggregations": {
+ "unique_connector_file_ids": {"buckets": [{"key": "c::a"}]},
+ "unique_document_ids": {"buckets": []},
+ "unique_filenames": {"buckets": []},
+ }
+ }
+
+ opensearch_client.search = AsyncMock(side_effect=fake_search)
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ file_ids, filenames, id_field = await connectors_api.get_synced_file_ids_for_connector(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ assert file_ids == ["c::a"]
+ assert id_field == "connector_file_id"
+ assert called_fields == ["connector_file_id", "connector_file_id.keyword"]
+
+
# ---------------------------------------------------------------------------
# connector_sync — bucket_filter reconciliation
# ---------------------------------------------------------------------------
diff --git a/tests/unit/connectors/test_connector_file_type_validation.py b/tests/unit/connectors/test_connector_file_type_validation.py
index 388b64e3a..8d0d98246 100644
--- a/tests/unit/connectors/test_connector_file_type_validation.py
+++ b/tests/unit/connectors/test_connector_file_type_validation.py
@@ -171,6 +171,98 @@ async def test_connector_check_duplicates():
]
+async def test_connector_check_duplicates_bucket_filter_is_existence_based():
+ """Bucket-kind connectors (Azure Blob/S3/COS) select whole buckets, not
+ individual files, so check-duplicates must accept bucket_filter and list
+ files from those buckets. Classification must be existence-based — any
+ blob already ingested under this connector_type is a duplicate — the same
+ semantics the filename-based OAuth check uses, NOT the modified-time
+ "changed" gate the real bucket_filter sync uses to silently skip unchanged
+ blobs. Otherwise re-selecting an up-to-date bucket reports zero duplicates
+ and the overwrite dialog never appears."""
+ import json
+
+ from fastapi.responses import JSONResponse
+
+ from api.connectors import ConnectorCheckDuplicatesBody, connector_check_duplicates
+
+ connector_service = MagicMock()
+ connection_manager = MagicMock()
+ connector_service.connection_manager = connection_manager
+
+ connection = MagicMock()
+ connection.connection_id = "conn-id"
+ connection.is_active = True
+ connection_manager.list_connections = AsyncMock(return_value=[connection])
+
+ connector = MagicMock()
+ connector.is_authenticated = True
+ connector.authenticate = AsyncMock(return_value=True)
+ connector.bucket_names = None
+ connector.list_files = AsyncMock(
+ return_value={
+ "files": [
+ # Already ingested (remote timestamp is irrelevant) -> duplicate.
+ {
+ "id": "bucket::already-ingested.pdf",
+ "name": "already-ingested.pdf",
+ "mimeType": "application/pdf",
+ "modified_time": "2026-01-01T00:00:00Z",
+ },
+ # Not ingested yet -> new.
+ {
+ "id": "bucket::new.pdf",
+ "name": "new.pdf",
+ "mimeType": "application/pdf",
+ "modified_time": "2026-07-13T00:00:00Z",
+ },
+ ]
+ }
+ )
+ connector_service.get_connector = AsyncMock(return_value=connector)
+
+ session_manager = MagicMock()
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={
+ "aggregations": {
+ "unique_connector_file_ids": {
+ "buckets": [{"key": "bucket::already-ingested.pdf"}],
+ },
+ "unique_document_ids": {"buckets": []},
+ "unique_filenames": {"buckets": []},
+ }
+ }
+ )
+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ user = MagicMock()
+ user.user_id = "user-id"
+ user.jwt_token = "jwt-token"
+
+ body = ConnectorCheckDuplicatesBody(
+ connection_id="conn-id",
+ bucket_filter=["bucket"],
+ )
+
+ response = await connector_check_duplicates(
+ connector_type="azure_blob",
+ body=body,
+ request=MagicMock(),
+ connector_service=connector_service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ assert isinstance(response, JSONResponse)
+ data = json.loads(response.body.decode())
+ assert data["duplicate_names"] == ["already-ingested.pdf"]
+ assert data["duplicate_count"] == 1
+ assert data["total_files"] == 2
+ assert [f["id"] for f in data["non_duplicate_files"]] == ["bucket::new.pdf"]
+
+
@pytest.mark.asyncio
async def test_connector_sync_skip_duplicates_returns_no_files_when_all_selected_are_duplicates(
monkeypatch,