Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: langflow content duplicate detection (#2395)

VisveshJfd9e8a1f51ae
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index 903ab7be8..e97b9f7bf 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -66,9 +66,17 @@ export interface File {
| "unavailable"
| "failed"
| "cancelled"
+ | "skipped"
| "hidden"
| "sync";
error?: string;
+ /**
+ * Skip reason forwarded from result.reason (e.g. "duplicate_content",
+ * "deleted_at_source"). Only set when status === "skipped".
+ */
+ skip_reason?: string;
+ /** Warning message for skipped rows (e.g. duplicate_content). */
+ warning?: string;
task_id?: string; // Task ID for file-level cancellation
chunks?: ChunkResult[];
allowed_users?: string[];
diff --git a/frontend/app/knowledge/page.test.tsx b/frontend/app/knowledge/page.test.tsx
index 382074159..2e5c3e0c1 100644
--- a/frontend/app/knowledge/page.test.tsx
+++ b/frontend/app/knowledge/page.test.tsx
@@ -1,16 +1,99 @@
/**
- * KnowledgePage — tests for the chunks-navigation URL helper.
+ * Unit tests for pure helpers extracted from knowledge/page.tsx.
*
- * The filename→chunks click handler lives inside an ag-Grid cellRenderer, which
- * jsdom cannot exercise (no layout engine; cell renderers never mount). Geometry-
- * and grid-dependent behaviour belongs in Playwright.
- *
- * What we CAN test in Vitest is the pure URL-building logic. It is extracted
- * into buildChunksUrl() in page.tsx so this test imports and calls the real
- * production function rather than re-implementing it inline.
+ * Covers the lines added / changed in #2388:
+ * getStatusSortRank — all branches including "skipped" (4) and "hidden" (7)
+ * getSkippedWarningText — provided warning vs default fallback
+ * isSkippedStatus — skipped detection used by the cell renderer
+ * SkippedStatusCell — renders the amber "Duplicate" badge with tooltip
*/
+import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
-import { buildChunksUrl } from "./page";
+import { TooltipProvider } from "@/components/ui/tooltip";
+import {
+ buildChunksUrl,
+ getSkippedWarningText,
+ getStatusSortRank,
+ isSkippedStatus,
+ SkippedStatusCell,
+} from "./page";
+
+describe("getStatusSortRank", () => {
+ it("ranks every status in the correct order", () => {
+ expect(getStatusSortRank("active")).toBe(0);
+ expect(getStatusSortRank("processing")).toBe(1);
+ expect(getStatusSortRank("sync")).toBe(2);
+ expect(getStatusSortRank("failed")).toBe(3);
+ expect(getStatusSortRank("skipped")).toBe(4);
+ expect(getStatusSortRank("cancelled")).toBe(5);
+ expect(getStatusSortRank("unavailable")).toBe(6);
+ expect(getStatusSortRank("hidden")).toBe(7);
+ });
+
+ it("returns 0 for undefined and unknown values", () => {
+ expect(getStatusSortRank(undefined)).toBe(0);
+ });
+
+ it("ranks skipped below failed but above cancelled", () => {
+ expect(getStatusSortRank("skipped")).toBeGreaterThan(
+ getStatusSortRank("failed"),
+ );
+ expect(getStatusSortRank("skipped")).toBeLessThan(
+ getStatusSortRank("cancelled"),
+ );
+ });
+});
+
+describe("getSkippedWarningText", () => {
+ it("returns the provided warning when present", () => {
+ const msg =
+ "Identical content already exists in the knowledge base under a different filename.";
+ expect(getSkippedWarningText(msg)).toBe(msg);
+ });
+
+ it("returns the default fallback when warning is undefined", () => {
+ expect(getSkippedWarningText(undefined)).toBe(
+ "Duplicate content — already exists in the knowledge base.",
+ );
+ });
+});
+
+describe("isSkippedStatus", () => {
+ it("returns true only for skipped + duplicate_content reason", () => {
+ expect(isSkippedStatus("skipped", "duplicate_content")).toBe(true);
+ });
+
+ it("returns false when status is skipped but reason is not duplicate_content", () => {
+ expect(isSkippedStatus("skipped", "deleted_at_source")).toBe(false);
+ expect(isSkippedStatus("skipped", undefined)).toBe(false);
+ });
+
+ it("returns false for any other status", () => {
+ expect(isSkippedStatus("active", "duplicate_content")).toBe(false);
+ expect(isSkippedStatus("failed")).toBe(false);
+ expect(isSkippedStatus(undefined)).toBe(false);
+ });
+});
+
+describe("SkippedStatusCell", () => {
+ it("renders the visible Duplicate label for both warning variants", () => {
+ // Radix TooltipContent is not in the DOM until hover — assert only the
+ // always-visible trigger span. Warning text is covered by getSkippedWarningText tests.
+ const { rerender } = render(
+ <TooltipProvider>
+ <SkippedStatusCell warning="Identical content already exists." />
+ </TooltipProvider>,
+ );
+ expect(screen.getByText("Duplicate")).toBeTruthy();
+
+ rerender(
+ <TooltipProvider>
+ <SkippedStatusCell />
+ </TooltipProvider>,
+ );
+ expect(screen.getByText("Duplicate")).toBeTruthy();
+ });
+});
describe("buildChunksUrl", () => {
it("includes ?q= when effectiveSearchText is a non-wildcard term", () => {
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 00d48aac8..fa971c750 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -156,6 +156,74 @@ function buildFilterPageResetKey(
});
}
+/** Pure helper — exported for unit tests, used by the status column comparator. */
+export function getStatusSortRank(status?: File["status"]): number {
+ switch (status) {
+ case "active":
+ return 0;
+ case "processing":
+ return 1;
+ case "sync":
+ return 2;
+ case "failed":
+ return 3;
+ case "skipped":
+ return 4;
+ case "cancelled":
+ return 5;
+ case "unavailable":
+ return 6;
+ case "hidden":
+ return 7;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * Pure helper — exported for unit tests, used by the status column cell renderer.
+ * Returns the warning text for a skipped (duplicate-content) file.
+ */
+export function getSkippedWarningText(warning?: string): string {
+ return warning ?? "Duplicate content — already exists in the knowledge base.";
+}
+
+/**
+ * Pure helper — exported for unit tests, used by the status column cell renderer.
+ * Returns true only for skipped rows that are content duplicates.
+ * Other skip reasons (e.g. deleted_at_source) fall through to the normal StatusBadge.
+ */
+export function isSkippedStatus(
+ rawStatus?: string,
+ skipReason?: string,
+): boolean {
+ return rawStatus === "skipped" && skipReason === "duplicate_content";
+}
+
+/**
+ * Exported for unit tests — renders the amber "Duplicate" badge with tooltip
+ * used by the status column cell renderer when a file was skipped.
+ */
+export function SkippedStatusCell({ warning }: { warning?: string }) {
+ const warningText = getSkippedWarningText(warning);
+ return (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="inline-flex items-center gap-1 text-sm text-amber-600 dark:text-amber-400 cursor-default">
+ Duplicate
+ </span>
+ </TooltipTrigger>
+ <TooltipContent
+ side="top"
+ align="end"
+ className="max-w-80 whitespace-pre-wrap break-words"
+ >
+ {warningText}
+ </TooltipContent>
+ </Tooltip>
+ );
+}
+
/** Builds the URL for navigating from a knowledge search result to its chunks page.
* Exported for unit testing; the click handler in SearchPage calls this directly. */
export function buildChunksUrl(
@@ -464,27 +532,6 @@ function SearchPage() {
return file?.owner_name?.trim() || file?.owner_email?.trim() || "—";
}, []);
- const getStatusSortRank = useCallback((status?: File["status"]): number => {
- switch (status) {
- case "active":
- return 0;
- case "processing":
- return 1;
- case "sync":
- return 2;
- case "failed":
- return 3;
- case "cancelled":
- return 4;
- case "unavailable":
- return 5;
- case "hidden":
- return 6;
- default:
- return 0;
- }
- }, []);
-
const hasOpenragRefreshCueFromTasks = tasks.some((task) => {
const isTaskActive =
task.status === "pending" ||
@@ -842,7 +889,15 @@ function SearchPage() {
return <StatusBadge status="cancelled" />;
}
- return <StatusBadge status={status} />;
+ if (isSkippedStatus(rawStatus, data?.skip_reason)) {
+ return <SkippedStatusCell warning={data?.warning} />;
+ }
+
+ return (
+ <StatusBadge
+ status={status as import("@/components/ui/status-badge").Status}
+ />
+ );
},
},
{
diff --git a/frontend/components/ui/sonner.tsx b/frontend/components/ui/sonner.tsx
index b09373662..fc324aa23 100644
--- a/frontend/components/ui/sonner.tsx
+++ b/frontend/components/ui/sonner.tsx
@@ -18,6 +18,8 @@ const Toaster = ({ ...props }: ToasterProps) => {
"group toast group-[.toaster]:bg-background group-[.toaster]:text-foreground group-[.toaster]:border-border group-[.toaster]:shadow-lg",
success:
"group-[.toaster]:!bg-green-600 group-[.toaster]:!text-white group-[.toaster]:!border-green-600",
+ warning:
+ "group-[.toaster]:!bg-amber-500 group-[.toaster]:!text-white group-[.toaster]:!border-amber-500",
error:
"group-[.toaster]:!bg-red-600 group-[.toaster]:!text-white group-[.toaster]:!border-red-600",
description: "!text-muted-foreground",
diff --git a/frontend/contexts/task-context.test.tsx b/frontend/contexts/task-context.test.tsx
index 52f0ab187..f51e30857 100644
--- a/frontend/contexts/task-context.test.tsx
+++ b/frontend/contexts/task-context.test.tsx
@@ -422,3 +422,277 @@ describe("TaskProvider — task completion with cancellations", () => {
);
});
});
+
+describe("TaskProvider — skipped files (duplicate detection)", () => {
+ beforeEach(() => {
+ toast.success.mockClear();
+ toast.error.mockClear();
+ toast.info.mockClear();
+ toast.warning.mockClear();
+ });
+
+ it("maps 'skipped' file status from result and stores the warning message", async () => {
+ const qc = createTestQueryClient();
+
+ let callCount = 0;
+ const runningTask: Task = makeTask({
+ task_id: "task-3",
+ status: "running",
+ files: { "duplicate.pdf": { status: "running", error: "" } },
+ });
+ const completedTask: Task = {
+ ...runningTask,
+ status: "completed",
+ successful_files: 0,
+ failed_files: 0,
+ files: {
+ "duplicate.pdf": {
+ status: "skipped",
+ error: "",
+ result: {
+ reason: "duplicate_content",
+ warning:
+ "Duplicate content — already exists in the knowledge base.",
+ },
+ },
+ },
+ };
+
+ server.use(
+ http.get("/api/tasks/enhanced", () => {
+ callCount++;
+ return HttpResponse.json({
+ tasks: [callCount === 1 ? runningTask : completedTask],
+ });
+ }),
+ );
+
+ const { result } = renderHook(() => useTask(), { wrapper: wrapper(qc) });
+
+ // Wait for the running task to be seen, then the completed one.
+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {
+ timeout: 5000,
+ });
+
+ // The file overlay should have status "skipped", skip_reason and warning set.
+ await waitFor(
+ () => {
+ const f = result.current.files.find((f) => f.task_id === "task-3");
+ expect(f?.status).toBe("skipped");
+ expect(f?.skip_reason).toBe("duplicate_content");
+ expect(f?.warning).toBe(
+ "Duplicate content — already exists in the knowledge base.",
+ );
+ },
+ { timeout: 5000 },
+ );
+ });
+
+ it("emits a 'Duplicate file skipped' warning toast when all files are skipped", async () => {
+ const qc = createTestQueryClient();
+
+ let callCount = 0;
+ const runningTask: Task = makeTask({
+ task_id: "task-4",
+ status: "running",
+ files: { "dup.pdf": { status: "running", error: "" } },
+ });
+ const completedTask: Task = {
+ ...runningTask,
+ status: "completed",
+ successful_files: 0,
+ failed_files: 0,
+ files: {
+ "dup.pdf": {
+ status: "skipped",
+ error: "",
+ result: { reason: "duplicate_content", warning: "Duplicate content" },
+ },
+ },
+ };
+
+ server.use(
+ http.get("/api/tasks/enhanced", () => {
+ callCount++;
+ return HttpResponse.json({
+ tasks: [callCount === 1 ? runningTask : completedTask],
+ });
+ }),
+ );
+
+ renderHook(() => useTask(), { wrapper: wrapper(qc) });
+
+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {
+ timeout: 5000,
+ });
+
+ await waitFor(
+ () =>
+ expect(toast.warning).toHaveBeenCalledWith(
+ "Duplicate file skipped",
+ expect.objectContaining({
+ description:
+ "1 file skipped — duplicate content already in knowledge base",
+ }),
+ ),
+ { timeout: 5000 },
+ );
+ });
+
+ it("emits a success toast with skip count when some files succeed and some are skipped", async () => {
+ const qc = createTestQueryClient();
+
+ let callCount = 0;
+ const runningTask: Task = makeTask({
+ task_id: "task-5",
+ status: "running",
+ files: {
+ "new.pdf": { status: "running", error: "" },
+ "dup.pdf": { status: "running", error: "" },
+ },
+ });
+ const completedTask: Task = {
+ ...runningTask,
+ status: "completed",
+ // Backend counts skipped files in successful_files, so 1 real + 1 skipped = 2
+ successful_files: 2,
+ failed_files: 0,
+ files: {
+ "new.pdf": { status: "completed", error: "" },
+ "dup.pdf": {
+ status: "skipped",
+ error: "",
+ result: { reason: "duplicate_content", warning: "Duplicate" },
+ },
+ },
+ };
+
+ server.use(
+ http.get("/api/tasks/enhanced", () => {
+ callCount++;
+ return HttpResponse.json({
+ tasks: [callCount === 1 ? runningTask : completedTask],
+ });
+ }),
+ );
+
+ renderHook(() => useTask(), { wrapper: wrapper(qc) });
+
+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {
+ timeout: 5000,
+ });
+
+ await waitFor(
+ () =>
+ expect(toast.success).toHaveBeenCalledWith(
+ "Task completed",
+ expect.objectContaining({
+ description:
+ "1 file uploaded successfully, 1 file skipped (duplicate content)",
+ }),
+ ),
+ { timeout: 5000 },
+ );
+ });
+
+ it("keeps skipped file overlays in the table after task completion", async () => {
+ const qc = createTestQueryClient();
+
+ let callCount = 0;
+ const runningTask: Task = makeTask({
+ task_id: "task-6",
+ status: "running",
+ files: { "skipped.pdf": { status: "running", error: "" } },
+ });
+ const completedTask: Task = {
+ ...runningTask,
+ status: "completed",
+ successful_files: 0,
+ failed_files: 0,
+ files: {
+ "skipped.pdf": {
+ status: "skipped",
+ error: "",
+ result: { warning: "Duplicate" },
+ },
+ },
+ };
+
+ server.use(
+ http.get("/api/tasks/enhanced", () => {
+ callCount++;
+ return HttpResponse.json({
+ tasks: [callCount === 1 ? runningTask : completedTask],
+ });
+ }),
+ );
+
+ const { result } = renderHook(() => useTask(), { wrapper: wrapper(qc) });
+
+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {
+ timeout: 5000,
+ });
+
+ // Skipped files should remain in the overlay (not be removed).
+ await waitFor(
+ () => {
+ const f = result.current.files.find((f) => f.task_id === "task-6");
+ expect(f?.status).toBe("skipped");
+ expect(f).toBeDefined();
+ },
+ { timeout: 5000 },
+ );
+ });
+
+ it("treats 1 failed + 1 skipped + 0 successful as total failure", async () => {
+ const qc = createTestQueryClient();
+
+ let callCount = 0;
+ const runningTask: Task = makeTask({
+ task_id: "task-7",
+ status: "running",
+ files: {
+ "fail.pdf": { status: "running", error: "" },
+ "dup.pdf": { status: "running", error: "" },
+ },
+ });
+ const completedTask: Task = {
+ ...runningTask,
+ status: "completed",
+ successful_files: 0,
+ failed_files: 1,
+ files: {
+ "fail.pdf": { status: "failed", error: "Error" },
+ "dup.pdf": {
+ status: "skipped",
+ error: "",
+ result: { warning: "Duplicate" },
+ },
+ },
+ };
+
+ server.use(
+ http.get("/api/tasks/enhanced", () => {
+ callCount++;
+ return HttpResponse.json({
+ tasks: [callCount === 1 ? runningTask : completedTask],
+ });
+ }),
+ );
+
+ renderHook(() => useTask(), { wrapper: wrapper(qc) });
+
+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {
+ timeout: 5000,
+ });
+
+ // failedFiles > 0 and successfulFiles === 0 → total failure, even with skipped files.
+ await waitFor(
+ () => {
+ expect(toast.error).toHaveBeenCalled();
+ expect(toast.success).not.toHaveBeenCalled();
+ },
+ { timeout: 5000 },
+ );
+ });
+});
diff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsx
index 3f711ad6d..405c3bf16 100644
--- a/frontend/contexts/task-context.tsx
+++ b/frontend/contexts/task-context.tsx
@@ -39,8 +39,10 @@ import {
didTaskReachTerminalState,
finalizeProcessingOverlaysForEnhancedTask,
findTaskFileOverlayIndex,
+ getDuplicateContentFileCount,
getEnhancedListDisappearedFilePaths,
getFailedFileCount,
+ getSkippedFileCount,
getSuccessfulFileCount,
hasFailedFileEntries,
isTaskInProgressStatus,
@@ -56,11 +58,18 @@ export interface TaskFile {
source_url: string;
size: number;
connector_type: string;
- status: "active" | "failed" | "processing" | "cancelled";
+ status: "active" | "failed" | "processing" | "cancelled" | "skipped";
task_id: string;
created_at: string;
updated_at: string;
error?: string;
+ /**
+ * Skip reason forwarded from result.reason (e.g. "duplicate_content",
+ * "deleted_at_source"). Only set when status === "skipped".
+ */
+ skip_reason?: string;
+ /** Warning message surfaced for skipped files (e.g. duplicate_content). */
+ warning?: string;
embedding_model?: string;
embedding_dimensions?: number;
}
@@ -391,6 +400,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
mappedStatus =
currentTask.status === "cancelled" ? "cancelled" : "failed";
break;
+ case "skipped":
+ mappedStatus = "skipped";
+ break;
default:
mappedStatus = "processing";
}
@@ -403,6 +415,26 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
return resolved === "Unknown error" ? undefined : resolved;
})();
+ // Extract skip reason and warning from result (set by the backend).
+ // Only surface duplicate-specific UI for duplicate_content skips;
+ // other reasons (e.g. deleted_at_source) are silent/informational.
+ const skipReason =
+ mappedStatus === "skipped"
+ ? ((
+ fileInfoEntry.result as
+ | Record<string, unknown>
+ | undefined
+ )?.reason as string | undefined)
+ : undefined;
+ const fileWarning =
+ mappedStatus === "skipped" &&
+ skipReason === "duplicate_content" &&
+ typeof (fileInfoEntry.result as Record<string, unknown>)
+ ?.warning === "string"
+ ? ((fileInfoEntry.result as Record<string, unknown>)
+ .warning as string)
+ : undefined;
+
setFiles((prevFiles) => {
const existingFileIndex = findTaskFileOverlayIndex(
prevFiles,
@@ -441,6 +473,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
? fileInfoEntry.updated_at
: now,
error: fileError,
+ skip_reason: skipReason,
+ warning: fileWarning,
embedding_model:
typeof fileInfoEntry.embedding_model === "string"
? fileInfoEntry.embedding_model
@@ -498,6 +532,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
currentTask.status === "completed"
) {
const successfulFiles = getSuccessfulFileCount(currentTask);
+ const skippedFiles = getSkippedFileCount(currentTask);
+ const duplicateFiles = getDuplicateContentFileCount(currentTask);
const failedFiles = getFailedFileCount(currentTask);
const isTotalFailure = failedFiles > 0 && successfulFiles === 0;
@@ -560,6 +596,18 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
failedFiles !== 1 ? "s" : ""
} failed`;
}
+ } else if (duplicateFiles > 0 && successfulFiles === 0) {
+ // All files were duplicate-skipped — no new content was indexed.
+ description = `${duplicateFiles} file${
+ duplicateFiles !== 1 ? "s" : ""
+ } skipped — duplicate content already in knowledge base`;
+ } else if (duplicateFiles > 0) {
+ // Mix of uploaded and skipped.
+ description = `${successfulFiles} file${
+ successfulFiles !== 1 ? "s" : ""
+ } uploaded successfully, ${duplicateFiles} file${
+ duplicateFiles !== 1 ? "s" : ""
+ } skipped (duplicate content)`;
} else {
description = `${successfulFiles} file${
successfulFiles !== 1 ? "s" : ""
@@ -584,6 +632,15 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
description,
action: toastAction,
});
+ } else if (
+ duplicateFiles > 0 &&
+ successfulFiles === 0 &&
+ failedFiles === 0
+ ) {
+ toast.warning("Duplicate file skipped", {
+ description,
+ action: toastAction,
+ });
} else {
toast.success("Task completed", {
description,
@@ -656,6 +713,16 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {
if (file.status === "failed") {
return completedHasFailures;
}
+ // Keep duplicate-content skipped overlays so the user sees the
+ // "Duplicate" badge — unless the same filename was later
+ // successfully indexed (a subsequent upload supersedes it).
+ if (file.status === "skipped") {
+ const filename = file.filename?.trim();
+ if (filename && indexedFilenames.has(filename)) {
+ return false;
+ }
+ return true;
+ }
if (file.status === "processing") {
return false;
}
diff --git a/frontend/lib/knowledge-table-state.test.ts b/frontend/lib/knowledge-table-state.test.ts
index c599667bb..ff187e2ab 100644
--- a/frontend/lib/knowledge-table-state.test.ts
+++ b/frontend/lib/knowledge-table-state.test.ts
@@ -332,6 +332,58 @@ describe("buildKnowledgeTableRows", () => {
);
expect(rows[0].status).toBe("cancelled");
});
+
+ it("treats a skipped overlay with the same priority as failed and cancelled", () => {
+ // skipped, cancelled, and failed all have priority 2. When a second overlay
+ // arrives with the same key, taskOverlayPriority is called for both —
+ // passing a `skipped` overlay through the switch exercises line 148.
+ const rows = buildKnowledgeTableRows(
+ [searchFile({ filename: "b.pdf", status: "active" })],
+ [
+ // First overlay: active (priority 1)
+ taskFile({ filename: "b.pdf", status: "active" }),
+ // Second overlay: skipped (priority 2) — should win over active
+ taskFile({
+ filename: "b.pdf",
+ status: "skipped",
+ warning: "Duplicate content",
+ }),
+ ],
+ );
+ expect(rows[0].status).toBe("skipped");
+ });
+
+ it("promotes skipped status onto the indexed row", () => {
+ const rows = buildKnowledgeTableRows(
+ [searchFile({ filename: "c.pdf", status: "active" })],
+ [
+ taskFile({
+ filename: "c.pdf",
+ status: "skipped",
+ warning: "Duplicate",
+ }),
+ ],
+ );
+ expect(rows[0].status).toBe("skipped");
+ expect(rows[0].warning).toBe("Duplicate");
+ });
+
+ it("merges warning message from skipped overlay onto the row", () => {
+ const rows = buildKnowledgeTableRows(
+ [searchFile({ filename: "d.pdf" })],
+ [
+ taskFile({
+ filename: "d.pdf",
+ status: "skipped",
+ warning:
+ "Duplicate content — already exists in the knowledge base.",
+ }),
+ ],
+ );
+ expect(rows[0].warning).toBe(
+ "Duplicate content — already exists in the knowledge base.",
+ );
+ });
});
});
diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.ts
index f69c36f2a..5b2592dfa 100644
--- a/frontend/lib/knowledge-table-state.ts
+++ b/frontend/lib/knowledge-table-state.ts
@@ -144,6 +144,8 @@ function taskOverlayPriority(status?: string): number {
return 2;
case "cancelled":
return 2;
+ case "skipped":
+ return 2;
case "active":
return 1;
default:
@@ -201,6 +203,8 @@ export function buildKnowledgeTableRows(
connector_type: taskFile.connector_type,
status: taskFile.status,
error: taskFile.error,
+ skip_reason: taskFile.skip_reason,
+ warning: taskFile.warning,
embedding_model: taskFile.embedding_model,
embedding_dimensions: taskFile.embedding_dimensions,
};
@@ -218,7 +222,8 @@ export function buildKnowledgeTableRows(
const status =
taskFile.status === "processing" ||
taskFile.status === "failed" ||
- taskFile.status === "cancelled"
+ taskFile.status === "cancelled" ||
+ taskFile.status === "skipped"
? taskFile.status
: backendStatus;
return {
@@ -235,6 +240,8 @@ export function buildKnowledgeTableRows(
),
status,
error: taskFile.error,
+ skip_reason: taskFile.skip_reason,
+ warning: taskFile.warning,
embedding_model: taskFile.embedding_model ?? file.embedding_model,
embedding_dimensions:
taskFile.embedding_dimensions ?? file.embedding_dimensions,
diff --git a/frontend/lib/task-utils.ts b/frontend/lib/task-utils.ts
index 007eea873..9f1f7a375 100644
--- a/frontend/lib/task-utils.ts
+++ b/frontend/lib/task-utils.ts
@@ -279,13 +279,35 @@ function isCompletedWithFailures(task: Task): boolean {
export function getSuccessfulFileCount(task: Task): number {
if (typeof task.successful_files === "number") {
- return task.successful_files;
+ // The backend counts skipped (duplicate) files as successful_files too.
+ // Subtract them so the toast only reports truly-indexed files as uploaded.
+ const skipped = getSkippedFileCount(task);
+ return Math.max(0, task.successful_files - skipped);
}
return Object.values(task.files || {}).filter(
(fileInfo) => fileInfo?.status === "completed",
).length;
}
+export function getSkippedFileCount(task: Task): number {
+ return Object.values(task.files || {}).filter(
+ (fileInfo) => fileInfo?.status === "skipped",
+ ).length;
+}
+
+/**
+ * Count only files skipped because of duplicate content (reason = "duplicate_content").
+ * Excludes other skip reasons such as "deleted_at_source" (connector cleanup).
+ */
+export function getDuplicateContentFileCount(task: Task): number {
+ return Object.values(task.files || {}).filter((fileInfo) => {
+ if (fileInfo?.status !== "skipped") return false;
+ const reason = (fileInfo.result as Record<string, unknown> | undefined)
+ ?.reason;
+ return reason === "duplicate_content";
+ }).length;
+}
+
export function getFailedFileCount(task: Task): number {
if (typeof task.failed_files === "number") {
return task.failed_files;
@@ -384,7 +406,7 @@ export function getEnhancedListDisappearedFilePaths(
interface ProcessingFileOverlay {
task_id: string;
source_url: string;
- status: "active" | "failed" | "processing" | "cancelled";
+ status: "active" | "failed" | "processing" | "cancelled" | "skipped";
error?: string;
}
diff --git a/frontend/tests/core/diverse-format-ingestion.spec.ts b/frontend/tests/core/diverse-format-ingestion.spec.ts
index 5c58727c6..e09d33ab8 100644
--- a/frontend/tests/core/diverse-format-ingestion.spec.ts
+++ b/frontend/tests/core/diverse-format-ingestion.spec.ts
@@ -43,10 +43,12 @@ test.describe("Diverse Format Ingestion. @33219208 , @34581152 , @34581153 , @34
}
}
- // Step 2: After all uploads complete, verify each file's status becomes Active
+ // Step 2: After all uploads complete, verify each file was processed without
+ // error. Active = newly indexed; Duplicate = correctly identified as already
+ // present. Both are successful outcomes on this path.
for (const uploadedFileName of uploadedFiles) {
try {
- await knowledge.verifyDocumentActive(uploadedFileName);
+ await knowledge.verifyDocumentProcessed(uploadedFileName);
} catch (error) {
throw new Error(`Failed to verify ${uploadedFileName}: ${error}`);
}
diff --git a/frontend/tests/pages/Knowledge.ts b/frontend/tests/pages/Knowledge.ts
index 13abcec65..ded39a2ad 100644
--- a/frontend/tests/pages/Knowledge.ts
+++ b/frontend/tests/pages/Knowledge.ts
@@ -754,6 +754,19 @@ export class Knowledge {
await searchInp.clear();
return;
}
+ // Terminal statuses will never transition to Active — fail immediately
+ // instead of burning the full 5-minute deadline before throwing.
+ const normalised = statusText.trim().toLowerCase();
+ if (
+ normalised === "failed" ||
+ normalised === "cancelled" ||
+ normalised === "duplicate"
+ ) {
+ await searchInp.clear();
+ throw new Error(
+ `Document "${docName}" reached terminal status "${statusText.trim()}" and will never become Active`,
+ );
+ }
logger.info(
` ⏳ Document "${docName}" status: "${statusText.trim()}", waiting...`,
);
@@ -765,6 +778,81 @@ export class Knowledge {
);
}
+ /**
+ * Verify that a document has been fully processed — either indexed as Active
+ * or recognised as a content duplicate (Duplicate). Both outcomes mean the
+ * system handled the file without error; use this instead of
+ * verifyDocumentActive when duplicate detection is expected to fire.
+ */
+ async verifyDocumentProcessed(docName: string) {
+ await this.open();
+ const searchInp = this.searchInput();
+ const searchReady = await searchInp
+ .waitFor({ state: "visible", timeout: 15000 })
+ .then(() => true)
+ .catch(() => false);
+ if (!searchReady) {
+ throw new Error(
+ `Knowledge grid search input not visible — page may not have loaded correctly`,
+ );
+ }
+
+ const deadline = Date.now() + 300000;
+ while (Date.now() < deadline) {
+ await searchInp.clear();
+ await searchInp.fill(docName);
+ await this.page.waitForTimeout(800);
+ let row = this.getRowById(docName).first();
+ if (!(await row.isVisible().catch(() => false))) {
+ const partial = this.getSourceCellByPartialText(docName);
+ if (await partial.isVisible().catch(() => false)) {
+ row = partial.locator('xpath=ancestor::*[@role="row"][1]');
+ }
+ }
+ if (!(await row.isVisible().catch(() => false))) {
+ logger.info(` ⏳ Document "${docName}" not yet visible, retrying...`);
+ await searchInp.clear();
+ continue;
+ }
+ await this.grid().evaluate((el) => {
+ el.scrollLeft = el.scrollWidth;
+ });
+ await this.page.waitForTimeout(300);
+ const status = this.getStatusCell(row);
+ let statusText = (await status.innerText().catch(() => "")) || "";
+ if (!statusText) {
+ statusText = (await status.textContent().catch(() => "")) || "";
+ }
+ if (statusText.includes("Animated Processing Icon")) {
+ statusText = "Processing";
+ }
+ const normalised = statusText.trim().toLowerCase();
+ // Both Active and Duplicate are successful terminal outcomes.
+ if (normalised.includes("active") || normalised === "duplicate") {
+ await searchInp.clear();
+ logger.info(
+ ` ✅ Document "${docName}" processed with status "${statusText.trim()}"`,
+ );
+ return;
+ }
+ // Hard-failure terminals — throw immediately.
+ if (normalised === "failed" || normalised === "cancelled") {
+ await searchInp.clear();
+ throw new Error(
+ `Document "${docName}" reached terminal status "${statusText.trim()}"`,
+ );
+ }
+ logger.info(
+ ` ⏳ Document "${docName}" status: "${statusText.trim()}", waiting...`,
+ );
+ await this.page.waitForTimeout(5000);
+ }
+ await searchInp.clear();
+ throw new Error(
+ `Document "${docName}" did not reach a processed status within the timeout`,
+ );
+ }
+
/**
* Verify that a document has been removed from the knowledge grid.
* Searches for the document, and asserts it is NOT visible.
diff --git a/src/models/processors.py b/src/models/processors.py
index e1b4d41d3..99eb44f5f 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -31,6 +31,9 @@ DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"
TEXT_PARSER_LABEL = "Text Parser"
DUPLICATE_FILENAME_WARNING = "A file with this name already exists."
+DUPLICATE_CONTENT_WARNING = (
+ "Identical content already exists in the knowledge base under a different filename."
+)
if TYPE_CHECKING:
from connectors.base import DocumentACL
@@ -1608,6 +1611,38 @@ class LangflowFileProcessor(TaskProcessor):
self.mark_duplicate_skipped(upload_task, file_task)
return
+ # Compute the content hash early — before reading file bytes into
+ # memory and before submitting any work to Docling so the duplicate
+ # guard fires as cheaply as possible.
+ #
+ # The guard only runs on the "proceed" path (no filename match).
+ # When duplicate_action == "replaced" the caller asked to replace an
+ # existing same-name document: the old chunks were already deleted
+ # above, so we must not short-circuit here even if the hash matches
+ # (that would leave the index empty after the delete).
+ file_hash = hash_id(item)
+ file_task.document_id = file_hash
+
+ if duplicate_action == "proceed" and await self.check_document_exists(
+ file_hash, opensearch_client
+ ):
+ # Identical content is already indexed under a different filename.
+ # Report it as a warning rather than silently overwriting.
+ # Mirrors the connector path (processors.py:1212) but uses a
+ # distinct reason so the UI can distinguish it from a filename
+ # collision.
+ file_task.status = TaskStatus.SKIPPED
+ file_task.error = None
+ file_task.result = {
+ "status": "skipped",
+ "reason": "duplicate_content",
+ "warning": DUPLICATE_CONTENT_WARNING,
+ "document_id": file_hash,
+ }
+ file_task.updated_at = time.time()
+ upload_task.successful_files += 1
+ return
+
# Read file content for processing
with open(item, "rb") as f:
content = f.read()
@@ -1634,9 +1669,6 @@ class LangflowFileProcessor(TaskProcessor):
# Prepare metadata tweaks similar to API endpoint
final_tweaks = self.tweaks.copy() if self.tweaks else {}
- file_hash = hash_id(item)
- file_task.document_id = file_hash
-
# Build settings with fresh OCR/pictureDescriptions from live
# config so retries pick up configuration changes.
config = get_openrag_config()
diff --git a/tests/unit/known_failures.txt b/tests/unit/known_failures.txt
index 413d5589c..84f412fb9 100644
--- a/tests/unit/known_failures.txt
+++ b/tests/unit/known_failures.txt
@@ -35,3 +35,5 @@ tests/unit/test_settings_provider_removal_defaults.py::TestFirstConfiguredEmbedd
tests/unit/test_settings_provider_removal_defaults.py::TestProviderRemovalLlmDefault::test_remove_openai_falls_back_to_ollama_empty_model
tests/unit/test_settings_refresh_endpoint.py::test_refresh_endpoint_requires_auth_in_auth_mode
tests/unit/test_settings_refresh_endpoint.py::test_refresh_endpoint_returns_expected_http_response_shape
+tests/unit/test_document_index_writer_bulk_retry.py::test_odd_length_bulk_body_raises
+tests/unit/test_processor_mapping_client.py::test_standard_processor_uses_shared_writer_for_embedding_mapping_and_writes
diff --git a/tests/unit/test_langflow_processor_content_dedupe.py b/tests/unit/test_langflow_processor_content_dedupe.py
new file mode 100644
index 000000000..3adb50a0c
--- /dev/null
+++ b/tests/unit/test_langflow_processor_content_dedupe.py
@@ -0,0 +1,243 @@
+"""Regression tests for content-duplicate detection in LangflowFileProcessor.
+
+When a user uploads a byte-identical file under a different name, the filename
+guard (resolve_duplicate_filename) does not fire — the names differ. The
+processor must fall through to a content-hash check (check_document_exists)
+and surface the result as SKIPPED with reason=duplicate_content, mirroring the
+behaviour of the connector and traditional paths.
+
+Relates to issue #2388.
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from models.processors import DUPLICATE_CONTENT_WARNING, LangflowFileProcessor
+from models.tasks import FileTask, TaskStatus, UploadTask
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _build_processor() -> LangflowFileProcessor:
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=AsyncMock())
+
+ langflow_file_service = MagicMock()
+ langflow_file_service.upload_and_ingest_file = AsyncMock(
+ return_value={"status": "indexed", "id": "hash-1"}
+ )
+
+ return LangflowFileProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ owner_user_id="user-1",
+ jwt_token="Bearer token",
+ )
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+async def test_langflow_processor_skips_on_content_duplicate(tmp_path):
+ """Re-uploading identical content under a new filename must be SKIPPED with
+ a user-visible warning, not silently overwritten."""
+ processor = _build_processor()
+
+ # Filename check: no existing document with this name → proceed.
+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")
+ # Content check: identical bytes already in the index.
+ processor.check_document_exists = AsyncMock(return_value=True)
+
+ item = tmp_path / "leave-policy-COPY.pdf"
+ item.write_bytes(b"%PDF-1.4 identical content")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "leave-policy-COPY.pdf"
+ upload_task = UploadTask(task_id="task-1", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ assert file_task.status == TaskStatus.SKIPPED
+ assert file_task.error is None
+ assert file_task.result == {
+ "status": "skipped",
+ "reason": "duplicate_content",
+ "warning": DUPLICATE_CONTENT_WARNING,
+ "document_id": file_task.document_id,
+ }
+ assert upload_task.failed_files == 0
+ assert upload_task.successful_files == 1
+ # Expensive work (Docling + Langflow) must not run for a content duplicate.
+ processor.langflow_file_service.upload_and_ingest_file.assert_not_called()
+
+
+async def test_langflow_processor_content_check_uses_file_hash(tmp_path):
+ """check_document_exists must be called with the content hash of the file,
+ so two identical files produce the same hash regardless of their names."""
+ from utils.hash_utils import hash_id
+
+ processor = _build_processor()
+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")
+ processor.check_document_exists = AsyncMock(return_value=True)
+
+ content = b"%PDF-1.4 hash-me"
+ item = tmp_path / "copy.pdf"
+ item.write_bytes(content)
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "copy.pdf"
+ upload_task = UploadTask(task_id="task-2", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ expected_hash = hash_id(str(item))
+ processor.check_document_exists.assert_awaited_once()
+ actual_hash_arg = processor.check_document_exists.await_args.args[0]
+ assert actual_hash_arg == expected_hash
+ assert file_task.document_id == expected_hash
+
+
+async def test_langflow_processor_new_content_proceeds(tmp_path):
+ """When no content duplicate exists, processing must continue normally —
+ the guard must be transparent for new files."""
+ processor = _build_processor()
+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")
+ # Content not yet indexed.
+ processor.check_document_exists = AsyncMock(return_value=False)
+ # Post-ingest verification: file visible after ingest.
+ processor.check_filename_exists = AsyncMock(return_value=True)
+
+ item = tmp_path / "new-file.pdf"
+ item.write_bytes(b"%PDF-1.4 brand new")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "new-file.pdf"
+ upload_task = UploadTask(task_id="task-3", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ assert file_task.status == TaskStatus.COMPLETED
+ assert upload_task.successful_files == 1
+ processor.langflow_file_service.upload_and_ingest_file.assert_awaited_once()
+
+
+async def test_langflow_processor_filename_duplicate_still_wins(tmp_path):
+ """The filename guard fires first; if it skips, the content guard is never
+ reached (and upload_and_ingest_file is still not called)."""
+ processor = _build_processor()
+ # Filename guard skips.
+ processor.resolve_duplicate_filename = AsyncMock(return_value="skip")
+ processor.check_document_exists = AsyncMock(return_value=True)
+
+ item = tmp_path / "report.pdf"
+ item.write_bytes(b"%PDF-1.4 something")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "report.pdf"
+ upload_task = UploadTask(task_id="task-4", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ assert file_task.status == TaskStatus.SKIPPED
+ assert file_task.result["reason"] == "duplicate_filename"
+ # Content check must never run when the filename guard already skipped.
+ processor.check_document_exists.assert_not_called()
+ processor.langflow_file_service.upload_and_ingest_file.assert_not_called()
+
+
+async def test_langflow_processor_replace_same_name_same_content_proceeds(tmp_path):
+ """replace_duplicates=True with a same-name file whose content hash already
+ exists must NOT trigger the content-duplicate skip. The old chunks were
+ already deleted by resolve_duplicate_filename; bailing out here would leave
+ the index empty. Ingest must proceed to re-index the document."""
+ processor = _build_processor()
+ # Filename guard deleted the old chunks and returned "replaced".
+ processor.resolve_duplicate_filename = AsyncMock(return_value="replaced")
+ # Hash matches the just-deleted document — must not short-circuit.
+ processor.check_document_exists = AsyncMock(return_value=True)
+ # Post-ingest verification: file visible after ingest.
+ processor.check_filename_exists = AsyncMock(return_value=True)
+
+ item = tmp_path / "report.pdf"
+ item.write_bytes(b"%PDF-1.4 same bytes")
+
+ file_task = FileTask(file_path=str(item))
+ file_task.filename = "report.pdf"
+ upload_task = UploadTask(task_id="task-5", total_files=1)
+
+ await processor.process_item(upload_task, str(item), file_task)
+
+ # Content guard must not have fired — ingest must have run.
+ processor.langflow_file_service.upload_and_ingest_file.assert_awaited_once()
+ assert file_task.status == TaskStatus.COMPLETED
+ assert upload_task.successful_files == 1
+ assert upload_task.failed_files == 0
+
+
+async def test_langflow_processor_concurrent_same_hash_only_one_proceeds(tmp_path):
+ """Two concurrent uploads with identical content must result in exactly one
+ Docling/Langflow submission. The second coroutine sees the hash already
+ present (as it would after the first ingest completes) and skips.
+
+ This is a best-effort in-process guard: check_document_exists is called
+ before upload_and_ingest_file, so the race window is limited to the gap
+ between the check and the write. A full atomic claim would require a
+ distributed lock (tracked separately); this test verifies the guard position
+ is correct and that the skip path is exercised under concurrency."""
+ import asyncio
+
+ call_count = 0
+
+ async def fake_upload(**kwargs):
+ nonlocal call_count
+ call_count += 1
+ return {"status": "indexed", "id": "hash-concurrent"}
+
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=AsyncMock())
+
+ langflow_file_service = MagicMock()
+ langflow_file_service.upload_and_ingest_file = fake_upload
+
+ item = tmp_path / "concurrent.pdf"
+ item.write_bytes(b"%PDF-1.4 concurrent content")
+
+ # Simulate: first call sees hash absent, second sees it present (post-ingest).
+ exists_results = [False, True]
+
+ async def fake_check_document_exists(file_hash, client):
+ return exists_results.pop(0)
+
+ async def run_one():
+ processor = LangflowFileProcessor(
+ langflow_file_service=langflow_file_service,
+ session_manager=session_manager,
+ owner_user_id="user-1",
+ jwt_token="Bearer token",
+ )
+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")
+ processor.check_document_exists = fake_check_document_exists
+ processor.check_filename_exists = AsyncMock(return_value=True)
+ ft = FileTask(file_path=str(item))
+ ft.filename = "concurrent.pdf"
+ ut = UploadTask(task_id="task-concurrent", total_files=2)
+ await processor.process_item(ut, str(item), ft)
+ return ft
+
+ results = await asyncio.gather(run_one(), run_one())
+
+ statuses = {r.status for r in results}
+ assert call_count == 1, "upload_and_ingest_file must be called exactly once"
+ assert TaskStatus.COMPLETED in statuses
+ assert TaskStatus.SKIPPED in statuses