Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: surface a failed duplicate check instead of ingesting anyway (#2434)

Rico Furtado494bd4ad4bb5
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsx
index ff43cda26..3cd432749 100644
--- a/frontend/app/upload/[provider]/page.tsx
+++ b/frontend/app/upload/[provider]/page.tsx
@@ -216,8 +216,11 @@ export default function UploadProviderPage() {
setDuplicateDialogOpen(true);
} catch (err) {
console.error("[Connector Sync] Duplicate check failed:", err);
- // Fallback: proceed without overwrite
- submitSync(connector, selectedFiles, false);
+ // Don't fall through to the sync: see shared-bucket-view. Ingesting
+ // without the answer silently skips every duplicate instead of asking.
+ toast.error("Could not check for existing files", {
+ description: "Nothing was ingested. Try again.",
+ });
} finally {
setIsCheckingDuplicates(false);
}
diff --git a/frontend/components/connectors/shared-bucket-view.test.tsx b/frontend/components/connectors/shared-bucket-view.test.tsx
new file mode 100644
index 000000000..d8cd9addc
--- /dev/null
+++ b/frontend/components/connectors/shared-bucket-view.test.tsx
@@ -0,0 +1,201 @@
+import { HttpResponse, http } from "msw";
+import { toast } from "sonner";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { server } from "@/test-utils/msw/server";
+import {
+ fireEvent,
+ renderWithProviders,
+ screen,
+ waitFor,
+} from "@/test-utils/render";
+import { SharedBucketView } from "./shared-bucket-view";
+
+const CHECK_DUPLICATES = "/api/connectors/ibm_cos/check-duplicates";
+
+/**
+ * A failed duplicate check must not fall through to the ingest.
+ *
+ * Proceeding silently takes the "skip duplicates" path: the sync runs, every
+ * colliding file is skipped with "a file with this name already exists", and
+ * the user never gets the choice the check exists to offer. From the outside
+ * that is indistinguishable from "there were no duplicates" — which is how it
+ * reaches us, reported as the duplicate dialog not appearing.
+ */
+
+const mutate = vi.fn();
+
+vi.mock("sonner", () => ({
+ toast: {
+ success: vi.fn(),
+ error: vi.fn(() => "toast-id"),
+ info: vi.fn(),
+ dismiss: vi.fn(),
+ },
+}));
+
+vi.mock("@/lib/analytics", () => ({
+ trackStartProcess: vi.fn(),
+ trackProcessFailure: vi.fn(),
+}));
+
+vi.mock("@/app/api/queries/useGetSettingsQuery", () => ({
+ useGetSettingsQuery: () => ({
+ data: { show_provider_ingest_settings: false },
+ }),
+}));
+
+vi.mock("@/hooks/useSessionIngestSettings", () => ({
+ useSessionIngestSettings: () => [
+ {
+ chunkSize: 1000,
+ chunkOverlap: 200,
+ ocr: false,
+ pictureDescriptions: false,
+ embeddingModel: "prod-embed",
+ },
+ vi.fn(),
+ ],
+}));
+
+function renderView() {
+ return renderWithProviders(
+ <SharedBucketView
+ connector={{ type: "ibm_cos", name: "IBM COS", connectionId: "conn-1" }}
+ buckets={[{ name: "docs", ingested_count: 0 }]}
+ isLoading={false}
+ onRefetch={vi.fn()}
+ invalidateQueryKey={["connectors"]}
+ syncMutation={{ mutate, isPending: false } as never}
+ addTask={vi.fn()}
+ onBack={vi.fn()}
+ onDone={vi.fn()}
+ resourceLabel="bucket"
+ resourceLabelPlural="buckets"
+ />,
+ { providers: ["auth"] },
+ );
+}
+
+function selectBucketAndIngest() {
+ const view = renderView();
+ fireEvent.click(screen.getByRole("checkbox", { name: "docs" }));
+ fireEvent.click(screen.getByRole("button", { name: /Ingest 1 Bucket/ }));
+ return view;
+}
+
+describe("SharedBucketView duplicate-check failures", () => {
+ beforeEach(() => {
+ mutate.mockReset();
+ vi.mocked(toast.error).mockClear();
+ vi.mocked(toast.dismiss).mockClear();
+ });
+
+ it("ingests nothing and says so when the check request rejects", async () => {
+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));
+
+ selectBucketAndIngest();
+
+ await waitFor(() =>
+ expect(toast.error).toHaveBeenCalledWith(
+ "Could not check for existing files",
+ expect.objectContaining({
+ description: expect.stringContaining("Nothing was ingested"),
+ }),
+ ),
+ );
+ expect(mutate).not.toHaveBeenCalled();
+ });
+
+ it("offers an escape hatch that ingests (skipping duplicates) when the check keeps failing", async () => {
+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));
+
+ selectBucketAndIngest();
+
+ await waitFor(() => expect(toast.error).toHaveBeenCalled());
+ expect(mutate).not.toHaveBeenCalled();
+
+ const [, options] = vi.mocked(toast.error).mock.calls[0];
+ const action = options?.action as unknown as { onClick: () => void };
+ action.onClick();
+
+ expect(mutate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({ replace_duplicates: false }),
+ }),
+ expect.anything(),
+ );
+ });
+
+ it("dismisses the escape-hatch toast once the bucket selection changes", async () => {
+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));
+
+ selectBucketAndIngest();
+
+ await waitFor(() => expect(toast.error).toHaveBeenCalled());
+ const toastId = vi.mocked(toast.error).mock.results[0].value;
+
+ fireEvent.click(screen.getByRole("checkbox", { name: "docs" }));
+
+ expect(toast.dismiss).toHaveBeenCalledWith(toastId);
+ });
+
+ it("dismisses the escape-hatch toast when the view unmounts", async () => {
+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));
+
+ const { unmount } = selectBucketAndIngest();
+
+ await waitFor(() => expect(toast.error).toHaveBeenCalled());
+ const toastId = vi.mocked(toast.error).mock.results[0].value;
+
+ unmount();
+
+ expect(toast.dismiss).toHaveBeenCalledWith(toastId);
+ });
+
+ it("ingests nothing and says so when the check times out upstream", async () => {
+ server.use(
+ http.post(CHECK_DUPLICATES, () =>
+ HttpResponse.json({ error: "timeout" }, { status: 504 }),
+ ),
+ );
+
+ selectBucketAndIngest();
+
+ await waitFor(() => expect(toast.error).toHaveBeenCalled());
+ expect(mutate).not.toHaveBeenCalled();
+ });
+
+ it("still ingests when the check succeeds and finds nothing", async () => {
+ server.use(
+ http.post(CHECK_DUPLICATES, () =>
+ HttpResponse.json({ duplicate_names: [], duplicate_count: 0 }),
+ ),
+ );
+
+ selectBucketAndIngest();
+
+ await waitFor(() => expect(mutate).toHaveBeenCalled());
+ expect(toast.error).not.toHaveBeenCalled();
+ });
+
+ it("opens the dialog instead of ingesting when duplicates are found", async () => {
+ server.use(
+ http.post(CHECK_DUPLICATES, () =>
+ HttpResponse.json({
+ duplicate_names: ["report.pdf"],
+ duplicate_count: 1,
+ duplicate_files: [
+ { id: "docs::report.pdf", name: "report.pdf", mimeType: "" },
+ ],
+ non_duplicate_files: [],
+ }),
+ ),
+ );
+
+ selectBucketAndIngest();
+
+ await waitFor(() => expect(screen.getByText(/report\.pdf/)).toBeTruthy());
+ expect(mutate).not.toHaveBeenCalled();
+ expect(toast.error).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsx
index 45caf19f0..57bcaf33c 100644
--- a/frontend/components/connectors/shared-bucket-view.tsx
+++ b/frontend/components/connectors/shared-bucket-view.tsx
@@ -101,6 +101,11 @@ export function SharedBucketView({
nonDuplicateFiles: BucketDuplicateFile[];
} | null>(null);
const isOverwriteConfirmedRef = useRef(false);
+ // Tracks the "ingest anyway" toast shown when the duplicate check fails, so
+ // it can be dismissed instead of left to fire its captured (and
+ // increasingly stale) request after the selection changes or the view
+ // goes away.
+ const checkFailedToastIdRef = useRef<string | number | null>(null);
useEffect(() => {
if (
@@ -120,6 +125,27 @@ export function SharedBucketView({
}
}, [buckets, initialSelectedBuckets, selectedBuckets.size]); // eslint-disable-line react-hooks/exhaustive-deps
+ // A changed selection or ingest setting means the "ingest anyway" toast's
+ // captured request no longer matches what the user is looking at, so drop
+ // it rather than let a later click submit the old choice.
+ useEffect(() => {
+ if (checkFailedToastIdRef.current !== null) {
+ toast.dismiss(checkFailedToastIdRef.current);
+ checkFailedToastIdRef.current = null;
+ }
+ }, [selectedBuckets, ingestSettings, showSharedToggle]);
+
+ // The toast outlives the component by default; dismiss it on unmount so it
+ // can't fire a sync whose onSuccess/onError callbacks (addTask, onDone)
+ // belong to a view that's no longer there.
+ useEffect(() => {
+ return () => {
+ if (checkFailedToastIdRef.current !== null) {
+ toast.dismiss(checkFailedToastIdRef.current);
+ }
+ };
+ }, []);
+
const invalidate = () => {
queryClient.invalidateQueries({ queryKey: invalidateQueryKey });
};
@@ -272,9 +298,31 @@ export function SharedBucketView({
setDuplicateDialogOpen(true);
} catch (err) {
console.error("[Bucket Sync] Duplicate check failed:", err);
- // Fallback: proceed with the normal full sync (backend still handles
- // new/changed reconciliation on its own).
- runBucketSync();
+ // Don't fall through to the sync by default. Proceeding would silently
+ // take the "skip duplicates" path — the ingest runs, every colliding
+ // file is skipped with "a file with this name already exists", and the
+ // user never got the choice the check exists to offer. That outcome is
+ // indistinguishable from "there were no duplicates", which is how it
+ // gets reported as the duplicate dialog not appearing.
+ //
+ // A deterministic timeout (the likeliest failure on a large bucket,
+ // since the check lists every blob in the selection) would otherwise
+ // leave no way to ingest that bucket from the UI at all, so the toast
+ // offers the old behavior as an explicit, conscious choice instead of a
+ // silent fallback.
+ checkFailedToastIdRef.current = toast.error(
+ "Could not check for existing files",
+ {
+ description:
+ "Nothing was ingested. Try again — if it keeps failing, the selected " +
+ `${resourceLabelPlural} may be too large to check. Ingesting anyway ` +
+ "skips files that already exist.",
+ action: {
+ label: "Ingest, skip existing",
+ onClick: () => runBucketSync(),
+ },
+ },
+ );
} finally {
setIsCheckingDuplicates(false);
}
diff --git a/frontend/components/file-browser-dialog.test.tsx b/frontend/components/file-browser-dialog.test.tsx
index 5fb9b9310..32a7c90fd 100644
--- a/frontend/components/file-browser-dialog.test.tsx
+++ b/frontend/components/file-browser-dialog.test.tsx
@@ -1,7 +1,12 @@
import { toast } from "sonner";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { IngestSettings } from "@/components/cloud-picker/types";
-import { fireEvent, renderWithProviders, screen } from "@/test-utils/render";
+import {
+ fireEvent,
+ renderWithProviders,
+ screen,
+ waitFor,
+} from "@/test-utils/render";
import { FileBrowserDialog } from "./file-browser-dialog";
/**
@@ -68,3 +73,62 @@ describe("FileBrowserDialog ingest validation", () => {
expect(mutateAsync).not.toHaveBeenCalled();
});
});
+
+/**
+ * A failed duplicate check must not fall through to the ingest. Proceeding
+ * silently takes the "skip duplicates" path: the sync runs, every colliding
+ * file is skipped with "a file with this name already exists", and the user
+ * never gets the choice the check exists to offer — indistinguishable from
+ * "there were no duplicates", and reported as the dialog not appearing.
+ */
+describe("FileBrowserDialog duplicate-check failures", () => {
+ beforeEach(() => {
+ mutateAsync.mockReset();
+ vi.mocked(toast.error).mockClear();
+ });
+
+ function ingest() {
+ renderDialog(baseSettings);
+ fireEvent.click(screen.getByText("report.pdf"));
+ fireEvent.click(screen.getByRole("button", { name: /Ingest 1 file/ }));
+ }
+
+ it("ingests nothing and says so when the check request rejects", async () => {
+ global.fetch = vi.fn().mockRejectedValue(new Error("network down"));
+
+ ingest();
+
+ await waitFor(() =>
+ expect(toast.error).toHaveBeenCalledWith(
+ "Could not check for existing files",
+ {
+ description: "Nothing was ingested. Try again.",
+ },
+ ),
+ );
+ expect(mutateAsync).not.toHaveBeenCalled();
+ });
+
+ it("ingests nothing and says so when the check returns an error status", async () => {
+ global.fetch = vi
+ .fn()
+ .mockResolvedValue({ ok: false, statusText: "Gateway Timeout" });
+
+ ingest();
+
+ await waitFor(() => expect(toast.error).toHaveBeenCalled());
+ expect(mutateAsync).not.toHaveBeenCalled();
+ });
+
+ it("still ingests when the check succeeds and finds nothing", async () => {
+ global.fetch = vi.fn().mockResolvedValue({
+ ok: true,
+ json: async () => ({ duplicate_names: [], duplicate_count: 0 }),
+ });
+
+ ingest();
+
+ await waitFor(() => expect(mutateAsync).toHaveBeenCalled());
+ expect(toast.error).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsx
index f7a1c3a3e..189281e2a 100644
--- a/frontend/components/file-browser-dialog.tsx
+++ b/frontend/components/file-browser-dialog.tsx
@@ -245,10 +245,11 @@ export function FileBrowserDialog({
setDuplicateDialogOpen(true);
} catch (err) {
console.error("[File Browser] Duplicate check failed:", err);
- // Fallback: proceed without overwrite (backend will still skip
- // exact-duplicate filenames on its own), unless we already know some
- // selected files are stale re-ingests.
- await submitSync(filesPayload, hasStale);
+ // Don't fall through to the sync: see shared-bucket-view. Ingesting
+ // without the answer silently skips every duplicate instead of asking.
+ toast.error("Could not check for existing files", {
+ description: "Nothing was ingested. Try again.",
+ });
} finally {
setIsCheckingDuplicates(false);
}