OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: surface a failed duplicate check instead of ingesting anyway (#2434)
Rico Furtado494bd4ad4bb5
diff --git a/frontend/app/upload/[provider]/page.tsx b/frontend/app/upload/[provider]/page.tsxindex ff43cda26..3cd432749 100644--- a/frontend/app/upload/[provider]/page.tsx+++ b/frontend/app/upload/[provider]/page.tsx@@ -216,8 +216,11 @@ export default function UploadProviderPage() {setDuplicateDialogOpen(true);} catch (err) {console.error("[Connector Sync] Duplicate check failed:", err);- // Fallback: proceed without overwrite- submitSync(connector, selectedFiles, false);+ // Don't fall through to the sync: see shared-bucket-view. Ingesting+ // without the answer silently skips every duplicate instead of asking.+ toast.error("Could not check for existing files", {+ description: "Nothing was ingested. Try again.",+ });} finally {setIsCheckingDuplicates(false);}diff --git a/frontend/components/connectors/shared-bucket-view.test.tsx b/frontend/components/connectors/shared-bucket-view.test.tsxnew file mode 100644index 000000000..d8cd9addc--- /dev/null+++ b/frontend/components/connectors/shared-bucket-view.test.tsx@@ -0,0 +1,201 @@+import { HttpResponse, http } from "msw";+import { toast } from "sonner";+import { beforeEach, describe, expect, it, vi } from "vitest";+import { server } from "@/test-utils/msw/server";+import {+ fireEvent,+ renderWithProviders,+ screen,+ waitFor,+} from "@/test-utils/render";+import { SharedBucketView } from "./shared-bucket-view";++const CHECK_DUPLICATES = "/api/connectors/ibm_cos/check-duplicates";++/**+ * A failed duplicate check must not fall through to the ingest.+ *+ * Proceeding silently takes the "skip duplicates" path: the sync runs, every+ * colliding file is skipped with "a file with this name already exists", and+ * the user never gets the choice the check exists to offer. From the outside+ * that is indistinguishable from "there were no duplicates" — which is how it+ * reaches us, reported as the duplicate dialog not appearing.+ */++const mutate = vi.fn();++vi.mock("sonner", () => ({+ toast: {+ success: vi.fn(),+ error: vi.fn(() => "toast-id"),+ info: vi.fn(),+ dismiss: vi.fn(),+ },+}));++vi.mock("@/lib/analytics", () => ({+ trackStartProcess: vi.fn(),+ trackProcessFailure: vi.fn(),+}));++vi.mock("@/app/api/queries/useGetSettingsQuery", () => ({+ useGetSettingsQuery: () => ({+ data: { show_provider_ingest_settings: false },+ }),+}));++vi.mock("@/hooks/useSessionIngestSettings", () => ({+ useSessionIngestSettings: () => [+ {+ chunkSize: 1000,+ chunkOverlap: 200,+ ocr: false,+ pictureDescriptions: false,+ embeddingModel: "prod-embed",+ },+ vi.fn(),+ ],+}));++function renderView() {+ return renderWithProviders(+ <SharedBucketView+ connector={{ type: "ibm_cos", name: "IBM COS", connectionId: "conn-1" }}+ buckets={[{ name: "docs", ingested_count: 0 }]}+ isLoading={false}+ onRefetch={vi.fn()}+ invalidateQueryKey={["connectors"]}+ syncMutation={{ mutate, isPending: false } as never}+ addTask={vi.fn()}+ onBack={vi.fn()}+ onDone={vi.fn()}+ resourceLabel="bucket"+ resourceLabelPlural="buckets"+ />,+ { providers: ["auth"] },+ );+}++function selectBucketAndIngest() {+ const view = renderView();+ fireEvent.click(screen.getByRole("checkbox", { name: "docs" }));+ fireEvent.click(screen.getByRole("button", { name: /Ingest 1 Bucket/ }));+ return view;+}++describe("SharedBucketView duplicate-check failures", () => {+ beforeEach(() => {+ mutate.mockReset();+ vi.mocked(toast.error).mockClear();+ vi.mocked(toast.dismiss).mockClear();+ });++ it("ingests nothing and says so when the check request rejects", async () => {+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));++ selectBucketAndIngest();++ await waitFor(() =>+ expect(toast.error).toHaveBeenCalledWith(+ "Could not check for existing files",+ expect.objectContaining({+ description: expect.stringContaining("Nothing was ingested"),+ }),+ ),+ );+ expect(mutate).not.toHaveBeenCalled();+ });++ it("offers an escape hatch that ingests (skipping duplicates) when the check keeps failing", async () => {+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));++ selectBucketAndIngest();++ await waitFor(() => expect(toast.error).toHaveBeenCalled());+ expect(mutate).not.toHaveBeenCalled();++ const [, options] = vi.mocked(toast.error).mock.calls[0];+ const action = options?.action as unknown as { onClick: () => void };+ action.onClick();++ expect(mutate).toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({ replace_duplicates: false }),+ }),+ expect.anything(),+ );+ });++ it("dismisses the escape-hatch toast once the bucket selection changes", async () => {+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));++ selectBucketAndIngest();++ await waitFor(() => expect(toast.error).toHaveBeenCalled());+ const toastId = vi.mocked(toast.error).mock.results[0].value;++ fireEvent.click(screen.getByRole("checkbox", { name: "docs" }));++ expect(toast.dismiss).toHaveBeenCalledWith(toastId);+ });++ it("dismisses the escape-hatch toast when the view unmounts", async () => {+ server.use(http.post(CHECK_DUPLICATES, () => HttpResponse.error()));++ const { unmount } = selectBucketAndIngest();++ await waitFor(() => expect(toast.error).toHaveBeenCalled());+ const toastId = vi.mocked(toast.error).mock.results[0].value;++ unmount();++ expect(toast.dismiss).toHaveBeenCalledWith(toastId);+ });++ it("ingests nothing and says so when the check times out upstream", async () => {+ server.use(+ http.post(CHECK_DUPLICATES, () =>+ HttpResponse.json({ error: "timeout" }, { status: 504 }),+ ),+ );++ selectBucketAndIngest();++ await waitFor(() => expect(toast.error).toHaveBeenCalled());+ expect(mutate).not.toHaveBeenCalled();+ });++ it("still ingests when the check succeeds and finds nothing", async () => {+ server.use(+ http.post(CHECK_DUPLICATES, () =>+ HttpResponse.json({ duplicate_names: [], duplicate_count: 0 }),+ ),+ );++ selectBucketAndIngest();++ await waitFor(() => expect(mutate).toHaveBeenCalled());+ expect(toast.error).not.toHaveBeenCalled();+ });++ it("opens the dialog instead of ingesting when duplicates are found", async () => {+ server.use(+ http.post(CHECK_DUPLICATES, () =>+ HttpResponse.json({+ duplicate_names: ["report.pdf"],+ duplicate_count: 1,+ duplicate_files: [+ { id: "docs::report.pdf", name: "report.pdf", mimeType: "" },+ ],+ non_duplicate_files: [],+ }),+ ),+ );++ selectBucketAndIngest();++ await waitFor(() => expect(screen.getByText(/report\.pdf/)).toBeTruthy());+ expect(mutate).not.toHaveBeenCalled();+ expect(toast.error).not.toHaveBeenCalled();+ });+});diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex 45caf19f0..57bcaf33c 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -101,6 +101,11 @@ export function SharedBucketView({nonDuplicateFiles: BucketDuplicateFile[];} | null>(null);const isOverwriteConfirmedRef = useRef(false);+ // Tracks the "ingest anyway" toast shown when the duplicate check fails, so+ // it can be dismissed instead of left to fire its captured (and+ // increasingly stale) request after the selection changes or the view+ // goes away.+ const checkFailedToastIdRef = useRef<string | number | null>(null);useEffect(() => {if (@@ -120,6 +125,27 @@ export function SharedBucketView({}}, [buckets, initialSelectedBuckets, selectedBuckets.size]); // eslint-disable-line react-hooks/exhaustive-deps+ // A changed selection or ingest setting means the "ingest anyway" toast's+ // captured request no longer matches what the user is looking at, so drop+ // it rather than let a later click submit the old choice.+ useEffect(() => {+ if (checkFailedToastIdRef.current !== null) {+ toast.dismiss(checkFailedToastIdRef.current);+ checkFailedToastIdRef.current = null;+ }+ }, [selectedBuckets, ingestSettings, showSharedToggle]);++ // The toast outlives the component by default; dismiss it on unmount so it+ // can't fire a sync whose onSuccess/onError callbacks (addTask, onDone)+ // belong to a view that's no longer there.+ useEffect(() => {+ return () => {+ if (checkFailedToastIdRef.current !== null) {+ toast.dismiss(checkFailedToastIdRef.current);+ }+ };+ }, []);+const invalidate = () => {queryClient.invalidateQueries({ queryKey: invalidateQueryKey });};@@ -272,9 +298,31 @@ export function SharedBucketView({setDuplicateDialogOpen(true);} catch (err) {console.error("[Bucket Sync] Duplicate check failed:", err);- // Fallback: proceed with the normal full sync (backend still handles- // new/changed reconciliation on its own).- runBucketSync();+ // Don't fall through to the sync by default. Proceeding would silently+ // take the "skip duplicates" path — the ingest runs, every colliding+ // file is skipped with "a file with this name already exists", and the+ // user never got the choice the check exists to offer. That outcome is+ // indistinguishable from "there were no duplicates", which is how it+ // gets reported as the duplicate dialog not appearing.+ //+ // A deterministic timeout (the likeliest failure on a large bucket,+ // since the check lists every blob in the selection) would otherwise+ // leave no way to ingest that bucket from the UI at all, so the toast+ // offers the old behavior as an explicit, conscious choice instead of a+ // silent fallback.+ checkFailedToastIdRef.current = toast.error(+ "Could not check for existing files",+ {+ description:+ "Nothing was ingested. Try again — if it keeps failing, the selected " ++ `${resourceLabelPlural} may be too large to check. Ingesting anyway ` ++ "skips files that already exist.",+ action: {+ label: "Ingest, skip existing",+ onClick: () => runBucketSync(),+ },+ },+ );} finally {setIsCheckingDuplicates(false);}diff --git a/frontend/components/file-browser-dialog.test.tsx b/frontend/components/file-browser-dialog.test.tsxindex 5fb9b9310..32a7c90fd 100644--- a/frontend/components/file-browser-dialog.test.tsx+++ b/frontend/components/file-browser-dialog.test.tsx@@ -1,7 +1,12 @@import { toast } from "sonner";import { beforeEach, describe, expect, it, vi } from "vitest";import type { IngestSettings } from "@/components/cloud-picker/types";-import { fireEvent, renderWithProviders, screen } from "@/test-utils/render";+import {+ fireEvent,+ renderWithProviders,+ screen,+ waitFor,+} from "@/test-utils/render";import { FileBrowserDialog } from "./file-browser-dialog";/**@@ -68,3 +73,62 @@ describe("FileBrowserDialog ingest validation", () => {expect(mutateAsync).not.toHaveBeenCalled();});});++/**+ * A failed duplicate check must not fall through to the ingest. Proceeding+ * silently takes the "skip duplicates" path: the sync runs, every colliding+ * file is skipped with "a file with this name already exists", and the user+ * never gets the choice the check exists to offer — indistinguishable from+ * "there were no duplicates", and reported as the dialog not appearing.+ */+describe("FileBrowserDialog duplicate-check failures", () => {+ beforeEach(() => {+ mutateAsync.mockReset();+ vi.mocked(toast.error).mockClear();+ });++ function ingest() {+ renderDialog(baseSettings);+ fireEvent.click(screen.getByText("report.pdf"));+ fireEvent.click(screen.getByRole("button", { name: /Ingest 1 file/ }));+ }++ it("ingests nothing and says so when the check request rejects", async () => {+ global.fetch = vi.fn().mockRejectedValue(new Error("network down"));++ ingest();++ await waitFor(() =>+ expect(toast.error).toHaveBeenCalledWith(+ "Could not check for existing files",+ {+ description: "Nothing was ingested. Try again.",+ },+ ),+ );+ expect(mutateAsync).not.toHaveBeenCalled();+ });++ it("ingests nothing and says so when the check returns an error status", async () => {+ global.fetch = vi+ .fn()+ .mockResolvedValue({ ok: false, statusText: "Gateway Timeout" });++ ingest();++ await waitFor(() => expect(toast.error).toHaveBeenCalled());+ expect(mutateAsync).not.toHaveBeenCalled();+ });++ it("still ingests when the check succeeds and finds nothing", async () => {+ global.fetch = vi.fn().mockResolvedValue({+ ok: true,+ json: async () => ({ duplicate_names: [], duplicate_count: 0 }),+ });++ ingest();++ await waitFor(() => expect(mutateAsync).toHaveBeenCalled());+ expect(toast.error).not.toHaveBeenCalled();+ });+});diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsxindex f7a1c3a3e..189281e2a 100644--- a/frontend/components/file-browser-dialog.tsx+++ b/frontend/components/file-browser-dialog.tsx@@ -245,10 +245,11 @@ export function FileBrowserDialog({setDuplicateDialogOpen(true);} catch (err) {console.error("[File Browser] Duplicate check failed:", err);- // Fallback: proceed without overwrite (backend will still skip- // exact-duplicate filenames on its own), unless we already know some- // selected files are stale re-ingests.- await submitSync(filesPayload, hasStale);+ // Don't fall through to the sync: see shared-bucket-view. Ingesting+ // without the answer silently skips every duplicate instead of asking.+ toast.error("Could not check for existing files", {+ description: "Nothing was ingested. Try again.",+ });} finally {setIsCheckingDuplicates(false);}