OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: langflow content duplicate detection (#2395)
VisveshJfd9e8a1f51ae
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex 903ab7be8..e97b9f7bf 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -66,9 +66,17 @@ export interface File {| "unavailable"| "failed"| "cancelled"+ | "skipped"| "hidden"| "sync";error?: string;+ /**+ * Skip reason forwarded from result.reason (e.g. "duplicate_content",+ * "deleted_at_source"). Only set when status === "skipped".+ */+ skip_reason?: string;+ /** Warning message for skipped rows (e.g. duplicate_content). */+ warning?: string;task_id?: string; // Task ID for file-level cancellationchunks?: ChunkResult[];allowed_users?: string[];diff --git a/frontend/app/knowledge/page.test.tsx b/frontend/app/knowledge/page.test.tsxindex 382074159..2e5c3e0c1 100644--- a/frontend/app/knowledge/page.test.tsx+++ b/frontend/app/knowledge/page.test.tsx@@ -1,16 +1,99 @@/**- * KnowledgePage — tests for the chunks-navigation URL helper.+ * Unit tests for pure helpers extracted from knowledge/page.tsx.*- * The filename→chunks click handler lives inside an ag-Grid cellRenderer, which- * jsdom cannot exercise (no layout engine; cell renderers never mount). Geometry-- * and grid-dependent behaviour belongs in Playwright.- *- * What we CAN test in Vitest is the pure URL-building logic. It is extracted- * into buildChunksUrl() in page.tsx so this test imports and calls the real- * production function rather than re-implementing it inline.+ * Covers the lines added / changed in #2388:+ * getStatusSortRank — all branches including "skipped" (4) and "hidden" (7)+ * getSkippedWarningText — provided warning vs default fallback+ * isSkippedStatus — skipped detection used by the cell renderer+ * SkippedStatusCell — renders the amber "Duplicate" badge with tooltip*/+import { render, screen } from "@testing-library/react";import { describe, expect, it } from "vitest";-import { buildChunksUrl } from "./page";+import { TooltipProvider } from "@/components/ui/tooltip";+import {+ buildChunksUrl,+ getSkippedWarningText,+ getStatusSortRank,+ isSkippedStatus,+ SkippedStatusCell,+} from "./page";++describe("getStatusSortRank", () => {+ it("ranks every status in the correct order", () => {+ expect(getStatusSortRank("active")).toBe(0);+ expect(getStatusSortRank("processing")).toBe(1);+ expect(getStatusSortRank("sync")).toBe(2);+ expect(getStatusSortRank("failed")).toBe(3);+ expect(getStatusSortRank("skipped")).toBe(4);+ expect(getStatusSortRank("cancelled")).toBe(5);+ expect(getStatusSortRank("unavailable")).toBe(6);+ expect(getStatusSortRank("hidden")).toBe(7);+ });++ it("returns 0 for undefined and unknown values", () => {+ expect(getStatusSortRank(undefined)).toBe(0);+ });++ it("ranks skipped below failed but above cancelled", () => {+ expect(getStatusSortRank("skipped")).toBeGreaterThan(+ getStatusSortRank("failed"),+ );+ expect(getStatusSortRank("skipped")).toBeLessThan(+ getStatusSortRank("cancelled"),+ );+ });+});++describe("getSkippedWarningText", () => {+ it("returns the provided warning when present", () => {+ const msg =+ "Identical content already exists in the knowledge base under a different filename.";+ expect(getSkippedWarningText(msg)).toBe(msg);+ });++ it("returns the default fallback when warning is undefined", () => {+ expect(getSkippedWarningText(undefined)).toBe(+ "Duplicate content — already exists in the knowledge base.",+ );+ });+});++describe("isSkippedStatus", () => {+ it("returns true only for skipped + duplicate_content reason", () => {+ expect(isSkippedStatus("skipped", "duplicate_content")).toBe(true);+ });++ it("returns false when status is skipped but reason is not duplicate_content", () => {+ expect(isSkippedStatus("skipped", "deleted_at_source")).toBe(false);+ expect(isSkippedStatus("skipped", undefined)).toBe(false);+ });++ it("returns false for any other status", () => {+ expect(isSkippedStatus("active", "duplicate_content")).toBe(false);+ expect(isSkippedStatus("failed")).toBe(false);+ expect(isSkippedStatus(undefined)).toBe(false);+ });+});++describe("SkippedStatusCell", () => {+ it("renders the visible Duplicate label for both warning variants", () => {+ // Radix TooltipContent is not in the DOM until hover — assert only the+ // always-visible trigger span. Warning text is covered by getSkippedWarningText tests.+ const { rerender } = render(+ <TooltipProvider>+ <SkippedStatusCell warning="Identical content already exists." />+ </TooltipProvider>,+ );+ expect(screen.getByText("Duplicate")).toBeTruthy();++ rerender(+ <TooltipProvider>+ <SkippedStatusCell />+ </TooltipProvider>,+ );+ expect(screen.getByText("Duplicate")).toBeTruthy();+ });+});describe("buildChunksUrl", () => {it("includes ?q= when effectiveSearchText is a non-wildcard term", () => {diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex 00d48aac8..fa971c750 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -156,6 +156,74 @@ function buildFilterPageResetKey(});}+/** Pure helper — exported for unit tests, used by the status column comparator. */+export function getStatusSortRank(status?: File["status"]): number {+ switch (status) {+ case "active":+ return 0;+ case "processing":+ return 1;+ case "sync":+ return 2;+ case "failed":+ return 3;+ case "skipped":+ return 4;+ case "cancelled":+ return 5;+ case "unavailable":+ return 6;+ case "hidden":+ return 7;+ default:+ return 0;+ }+}++/**+ * Pure helper — exported for unit tests, used by the status column cell renderer.+ * Returns the warning text for a skipped (duplicate-content) file.+ */+export function getSkippedWarningText(warning?: string): string {+ return warning ?? "Duplicate content — already exists in the knowledge base.";+}++/**+ * Pure helper — exported for unit tests, used by the status column cell renderer.+ * Returns true only for skipped rows that are content duplicates.+ * Other skip reasons (e.g. deleted_at_source) fall through to the normal StatusBadge.+ */+export function isSkippedStatus(+ rawStatus?: string,+ skipReason?: string,+): boolean {+ return rawStatus === "skipped" && skipReason === "duplicate_content";+}++/**+ * Exported for unit tests — renders the amber "Duplicate" badge with tooltip+ * used by the status column cell renderer when a file was skipped.+ */+export function SkippedStatusCell({ warning }: { warning?: string }) {+ const warningText = getSkippedWarningText(warning);+ return (+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="inline-flex items-center gap-1 text-sm text-amber-600 dark:text-amber-400 cursor-default">+ Duplicate+ </span>+ </TooltipTrigger>+ <TooltipContent+ side="top"+ align="end"+ className="max-w-80 whitespace-pre-wrap break-words"+ >+ {warningText}+ </TooltipContent>+ </Tooltip>+ );+}+/** Builds the URL for navigating from a knowledge search result to its chunks page.* Exported for unit testing; the click handler in SearchPage calls this directly. */export function buildChunksUrl(@@ -464,27 +532,6 @@ function SearchPage() {return file?.owner_name?.trim() || file?.owner_email?.trim() || "—";}, []);- const getStatusSortRank = useCallback((status?: File["status"]): number => {- switch (status) {- case "active":- return 0;- case "processing":- return 1;- case "sync":- return 2;- case "failed":- return 3;- case "cancelled":- return 4;- case "unavailable":- return 5;- case "hidden":- return 6;- default:- return 0;- }- }, []);-const hasOpenragRefreshCueFromTasks = tasks.some((task) => {const isTaskActive =task.status === "pending" ||@@ -842,7 +889,15 @@ function SearchPage() {return <StatusBadge status="cancelled" />;}- return <StatusBadge status={status} />;+ if (isSkippedStatus(rawStatus, data?.skip_reason)) {+ return <SkippedStatusCell warning={data?.warning} />;+ }++ return (+ <StatusBadge+ status={status as import("@/components/ui/status-badge").Status}+ />+ );},},{diff --git a/frontend/components/ui/sonner.tsx b/frontend/components/ui/sonner.tsxindex b09373662..fc324aa23 100644--- a/frontend/components/ui/sonner.tsx+++ b/frontend/components/ui/sonner.tsx@@ -18,6 +18,8 @@ const Toaster = ({ ...props }: ToasterProps) => {"group toast group-[.toaster]:bg-background group-[.toaster]:text-foreground group-[.toaster]:border-border group-[.toaster]:shadow-lg",success:"group-[.toaster]:!bg-green-600 group-[.toaster]:!text-white group-[.toaster]:!border-green-600",+ warning:+ "group-[.toaster]:!bg-amber-500 group-[.toaster]:!text-white group-[.toaster]:!border-amber-500",error:"group-[.toaster]:!bg-red-600 group-[.toaster]:!text-white group-[.toaster]:!border-red-600",description: "!text-muted-foreground",diff --git a/frontend/contexts/task-context.test.tsx b/frontend/contexts/task-context.test.tsxindex 52f0ab187..f51e30857 100644--- a/frontend/contexts/task-context.test.tsx+++ b/frontend/contexts/task-context.test.tsx@@ -422,3 +422,277 @@ describe("TaskProvider — task completion with cancellations", () => {);});});++describe("TaskProvider — skipped files (duplicate detection)", () => {+ beforeEach(() => {+ toast.success.mockClear();+ toast.error.mockClear();+ toast.info.mockClear();+ toast.warning.mockClear();+ });++ it("maps 'skipped' file status from result and stores the warning message", async () => {+ const qc = createTestQueryClient();++ let callCount = 0;+ const runningTask: Task = makeTask({+ task_id: "task-3",+ status: "running",+ files: { "duplicate.pdf": { status: "running", error: "" } },+ });+ const completedTask: Task = {+ ...runningTask,+ status: "completed",+ successful_files: 0,+ failed_files: 0,+ files: {+ "duplicate.pdf": {+ status: "skipped",+ error: "",+ result: {+ reason: "duplicate_content",+ warning:+ "Duplicate content — already exists in the knowledge base.",+ },+ },+ },+ };++ server.use(+ http.get("/api/tasks/enhanced", () => {+ callCount++;+ return HttpResponse.json({+ tasks: [callCount === 1 ? runningTask : completedTask],+ });+ }),+ );++ const { result } = renderHook(() => useTask(), { wrapper: wrapper(qc) });++ // Wait for the running task to be seen, then the completed one.+ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {+ timeout: 5000,+ });++ // The file overlay should have status "skipped", skip_reason and warning set.+ await waitFor(+ () => {+ const f = result.current.files.find((f) => f.task_id === "task-3");+ expect(f?.status).toBe("skipped");+ expect(f?.skip_reason).toBe("duplicate_content");+ expect(f?.warning).toBe(+ "Duplicate content — already exists in the knowledge base.",+ );+ },+ { timeout: 5000 },+ );+ });++ it("emits a 'Duplicate file skipped' warning toast when all files are skipped", async () => {+ const qc = createTestQueryClient();++ let callCount = 0;+ const runningTask: Task = makeTask({+ task_id: "task-4",+ status: "running",+ files: { "dup.pdf": { status: "running", error: "" } },+ });+ const completedTask: Task = {+ ...runningTask,+ status: "completed",+ successful_files: 0,+ failed_files: 0,+ files: {+ "dup.pdf": {+ status: "skipped",+ error: "",+ result: { reason: "duplicate_content", warning: "Duplicate content" },+ },+ },+ };++ server.use(+ http.get("/api/tasks/enhanced", () => {+ callCount++;+ return HttpResponse.json({+ tasks: [callCount === 1 ? runningTask : completedTask],+ });+ }),+ );++ renderHook(() => useTask(), { wrapper: wrapper(qc) });++ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {+ timeout: 5000,+ });++ await waitFor(+ () =>+ expect(toast.warning).toHaveBeenCalledWith(+ "Duplicate file skipped",+ expect.objectContaining({+ description:+ "1 file skipped — duplicate content already in knowledge base",+ }),+ ),+ { timeout: 5000 },+ );+ });++ it("emits a success toast with skip count when some files succeed and some are skipped", async () => {+ const qc = createTestQueryClient();++ let callCount = 0;+ const runningTask: Task = makeTask({+ task_id: "task-5",+ status: "running",+ files: {+ "new.pdf": { status: "running", error: "" },+ "dup.pdf": { status: "running", error: "" },+ },+ });+ const completedTask: Task = {+ ...runningTask,+ status: "completed",+ // Backend counts skipped files in successful_files, so 1 real + 1 skipped = 2+ successful_files: 2,+ failed_files: 0,+ files: {+ "new.pdf": { status: "completed", error: "" },+ "dup.pdf": {+ status: "skipped",+ error: "",+ result: { reason: "duplicate_content", warning: "Duplicate" },+ },+ },+ };++ server.use(+ http.get("/api/tasks/enhanced", () => {+ callCount++;+ return HttpResponse.json({+ tasks: [callCount === 1 ? runningTask : completedTask],+ });+ }),+ );++ renderHook(() => useTask(), { wrapper: wrapper(qc) });++ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {+ timeout: 5000,+ });++ await waitFor(+ () =>+ expect(toast.success).toHaveBeenCalledWith(+ "Task completed",+ expect.objectContaining({+ description:+ "1 file uploaded successfully, 1 file skipped (duplicate content)",+ }),+ ),+ { timeout: 5000 },+ );+ });++ it("keeps skipped file overlays in the table after task completion", async () => {+ const qc = createTestQueryClient();++ let callCount = 0;+ const runningTask: Task = makeTask({+ task_id: "task-6",+ status: "running",+ files: { "skipped.pdf": { status: "running", error: "" } },+ });+ const completedTask: Task = {+ ...runningTask,+ status: "completed",+ successful_files: 0,+ failed_files: 0,+ files: {+ "skipped.pdf": {+ status: "skipped",+ error: "",+ result: { warning: "Duplicate" },+ },+ },+ };++ server.use(+ http.get("/api/tasks/enhanced", () => {+ callCount++;+ return HttpResponse.json({+ tasks: [callCount === 1 ? runningTask : completedTask],+ });+ }),+ );++ const { result } = renderHook(() => useTask(), { wrapper: wrapper(qc) });++ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {+ timeout: 5000,+ });++ // Skipped files should remain in the overlay (not be removed).+ await waitFor(+ () => {+ const f = result.current.files.find((f) => f.task_id === "task-6");+ expect(f?.status).toBe("skipped");+ expect(f).toBeDefined();+ },+ { timeout: 5000 },+ );+ });++ it("treats 1 failed + 1 skipped + 0 successful as total failure", async () => {+ const qc = createTestQueryClient();++ let callCount = 0;+ const runningTask: Task = makeTask({+ task_id: "task-7",+ status: "running",+ files: {+ "fail.pdf": { status: "running", error: "" },+ "dup.pdf": { status: "running", error: "" },+ },+ });+ const completedTask: Task = {+ ...runningTask,+ status: "completed",+ successful_files: 0,+ failed_files: 1,+ files: {+ "fail.pdf": { status: "failed", error: "Error" },+ "dup.pdf": {+ status: "skipped",+ error: "",+ result: { warning: "Duplicate" },+ },+ },+ };++ server.use(+ http.get("/api/tasks/enhanced", () => {+ callCount++;+ return HttpResponse.json({+ tasks: [callCount === 1 ? runningTask : completedTask],+ });+ }),+ );++ renderHook(() => useTask(), { wrapper: wrapper(qc) });++ await waitFor(() => expect(callCount).toBeGreaterThanOrEqual(2), {+ timeout: 5000,+ });++ // failedFiles > 0 and successfulFiles === 0 → total failure, even with skipped files.+ await waitFor(+ () => {+ expect(toast.error).toHaveBeenCalled();+ expect(toast.success).not.toHaveBeenCalled();+ },+ { timeout: 5000 },+ );+ });+});diff --git a/frontend/contexts/task-context.tsx b/frontend/contexts/task-context.tsxindex 3f711ad6d..405c3bf16 100644--- a/frontend/contexts/task-context.tsx+++ b/frontend/contexts/task-context.tsx@@ -39,8 +39,10 @@ import {didTaskReachTerminalState,finalizeProcessingOverlaysForEnhancedTask,findTaskFileOverlayIndex,+ getDuplicateContentFileCount,getEnhancedListDisappearedFilePaths,getFailedFileCount,+ getSkippedFileCount,getSuccessfulFileCount,hasFailedFileEntries,isTaskInProgressStatus,@@ -56,11 +58,18 @@ export interface TaskFile {source_url: string;size: number;connector_type: string;- status: "active" | "failed" | "processing" | "cancelled";+ status: "active" | "failed" | "processing" | "cancelled" | "skipped";task_id: string;created_at: string;updated_at: string;error?: string;+ /**+ * Skip reason forwarded from result.reason (e.g. "duplicate_content",+ * "deleted_at_source"). Only set when status === "skipped".+ */+ skip_reason?: string;+ /** Warning message surfaced for skipped files (e.g. duplicate_content). */+ warning?: string;embedding_model?: string;embedding_dimensions?: number;}@@ -391,6 +400,9 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {mappedStatus =currentTask.status === "cancelled" ? "cancelled" : "failed";break;+ case "skipped":+ mappedStatus = "skipped";+ break;default:mappedStatus = "processing";}@@ -403,6 +415,26 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {return resolved === "Unknown error" ? undefined : resolved;})();+ // Extract skip reason and warning from result (set by the backend).+ // Only surface duplicate-specific UI for duplicate_content skips;+ // other reasons (e.g. deleted_at_source) are silent/informational.+ const skipReason =+ mappedStatus === "skipped"+ ? ((+ fileInfoEntry.result as+ | Record<string, unknown>+ | undefined+ )?.reason as string | undefined)+ : undefined;+ const fileWarning =+ mappedStatus === "skipped" &&+ skipReason === "duplicate_content" &&+ typeof (fileInfoEntry.result as Record<string, unknown>)+ ?.warning === "string"+ ? ((fileInfoEntry.result as Record<string, unknown>)+ .warning as string)+ : undefined;+setFiles((prevFiles) => {const existingFileIndex = findTaskFileOverlayIndex(prevFiles,@@ -441,6 +473,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {? fileInfoEntry.updated_at: now,error: fileError,+ skip_reason: skipReason,+ warning: fileWarning,embedding_model:typeof fileInfoEntry.embedding_model === "string"? fileInfoEntry.embedding_model@@ -498,6 +532,8 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {currentTask.status === "completed") {const successfulFiles = getSuccessfulFileCount(currentTask);+ const skippedFiles = getSkippedFileCount(currentTask);+ const duplicateFiles = getDuplicateContentFileCount(currentTask);const failedFiles = getFailedFileCount(currentTask);const isTotalFailure = failedFiles > 0 && successfulFiles === 0;@@ -560,6 +596,18 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {failedFiles !== 1 ? "s" : ""} failed`;}+ } else if (duplicateFiles > 0 && successfulFiles === 0) {+ // All files were duplicate-skipped — no new content was indexed.+ description = `${duplicateFiles} file${+ duplicateFiles !== 1 ? "s" : ""+ } skipped — duplicate content already in knowledge base`;+ } else if (duplicateFiles > 0) {+ // Mix of uploaded and skipped.+ description = `${successfulFiles} file${+ successfulFiles !== 1 ? "s" : ""+ } uploaded successfully, ${duplicateFiles} file${+ duplicateFiles !== 1 ? "s" : ""+ } skipped (duplicate content)`;} else {description = `${successfulFiles} file${successfulFiles !== 1 ? "s" : ""@@ -584,6 +632,15 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {description,action: toastAction,});+ } else if (+ duplicateFiles > 0 &&+ successfulFiles === 0 &&+ failedFiles === 0+ ) {+ toast.warning("Duplicate file skipped", {+ description,+ action: toastAction,+ });} else {toast.success("Task completed", {description,@@ -656,6 +713,16 @@ export function TaskProvider({ children }: { children: React.ReactNode }) {if (file.status === "failed") {return completedHasFailures;}+ // Keep duplicate-content skipped overlays so the user sees the+ // "Duplicate" badge — unless the same filename was later+ // successfully indexed (a subsequent upload supersedes it).+ if (file.status === "skipped") {+ const filename = file.filename?.trim();+ if (filename && indexedFilenames.has(filename)) {+ return false;+ }+ return true;+ }if (file.status === "processing") {return false;}diff --git a/frontend/lib/knowledge-table-state.test.ts b/frontend/lib/knowledge-table-state.test.tsindex c599667bb..ff187e2ab 100644--- a/frontend/lib/knowledge-table-state.test.ts+++ b/frontend/lib/knowledge-table-state.test.ts@@ -332,6 +332,58 @@ describe("buildKnowledgeTableRows", () => {);expect(rows[0].status).toBe("cancelled");});++ it("treats a skipped overlay with the same priority as failed and cancelled", () => {+ // skipped, cancelled, and failed all have priority 2. When a second overlay+ // arrives with the same key, taskOverlayPriority is called for both —+ // passing a `skipped` overlay through the switch exercises line 148.+ const rows = buildKnowledgeTableRows(+ [searchFile({ filename: "b.pdf", status: "active" })],+ [+ // First overlay: active (priority 1)+ taskFile({ filename: "b.pdf", status: "active" }),+ // Second overlay: skipped (priority 2) — should win over active+ taskFile({+ filename: "b.pdf",+ status: "skipped",+ warning: "Duplicate content",+ }),+ ],+ );+ expect(rows[0].status).toBe("skipped");+ });++ it("promotes skipped status onto the indexed row", () => {+ const rows = buildKnowledgeTableRows(+ [searchFile({ filename: "c.pdf", status: "active" })],+ [+ taskFile({+ filename: "c.pdf",+ status: "skipped",+ warning: "Duplicate",+ }),+ ],+ );+ expect(rows[0].status).toBe("skipped");+ expect(rows[0].warning).toBe("Duplicate");+ });++ it("merges warning message from skipped overlay onto the row", () => {+ const rows = buildKnowledgeTableRows(+ [searchFile({ filename: "d.pdf" })],+ [+ taskFile({+ filename: "d.pdf",+ status: "skipped",+ warning:+ "Duplicate content — already exists in the knowledge base.",+ }),+ ],+ );+ expect(rows[0].warning).toBe(+ "Duplicate content — already exists in the knowledge base.",+ );+ });});});diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.tsindex f69c36f2a..5b2592dfa 100644--- a/frontend/lib/knowledge-table-state.ts+++ b/frontend/lib/knowledge-table-state.ts@@ -144,6 +144,8 @@ function taskOverlayPriority(status?: string): number {return 2;case "cancelled":return 2;+ case "skipped":+ return 2;case "active":return 1;default:@@ -201,6 +203,8 @@ export function buildKnowledgeTableRows(connector_type: taskFile.connector_type,status: taskFile.status,error: taskFile.error,+ skip_reason: taskFile.skip_reason,+ warning: taskFile.warning,embedding_model: taskFile.embedding_model,embedding_dimensions: taskFile.embedding_dimensions,};@@ -218,7 +222,8 @@ export function buildKnowledgeTableRows(const status =taskFile.status === "processing" ||taskFile.status === "failed" ||- taskFile.status === "cancelled"+ taskFile.status === "cancelled" ||+ taskFile.status === "skipped"? taskFile.status: backendStatus;return {@@ -235,6 +240,8 @@ export function buildKnowledgeTableRows(),status,error: taskFile.error,+ skip_reason: taskFile.skip_reason,+ warning: taskFile.warning,embedding_model: taskFile.embedding_model ?? file.embedding_model,embedding_dimensions:taskFile.embedding_dimensions ?? file.embedding_dimensions,diff --git a/frontend/lib/task-utils.ts b/frontend/lib/task-utils.tsindex 007eea873..9f1f7a375 100644--- a/frontend/lib/task-utils.ts+++ b/frontend/lib/task-utils.ts@@ -279,13 +279,35 @@ function isCompletedWithFailures(task: Task): boolean {export function getSuccessfulFileCount(task: Task): number {if (typeof task.successful_files === "number") {- return task.successful_files;+ // The backend counts skipped (duplicate) files as successful_files too.+ // Subtract them so the toast only reports truly-indexed files as uploaded.+ const skipped = getSkippedFileCount(task);+ return Math.max(0, task.successful_files - skipped);}return Object.values(task.files || {}).filter((fileInfo) => fileInfo?.status === "completed",).length;}+export function getSkippedFileCount(task: Task): number {+ return Object.values(task.files || {}).filter(+ (fileInfo) => fileInfo?.status === "skipped",+ ).length;+}++/**+ * Count only files skipped because of duplicate content (reason = "duplicate_content").+ * Excludes other skip reasons such as "deleted_at_source" (connector cleanup).+ */+export function getDuplicateContentFileCount(task: Task): number {+ return Object.values(task.files || {}).filter((fileInfo) => {+ if (fileInfo?.status !== "skipped") return false;+ const reason = (fileInfo.result as Record<string, unknown> | undefined)+ ?.reason;+ return reason === "duplicate_content";+ }).length;+}+export function getFailedFileCount(task: Task): number {if (typeof task.failed_files === "number") {return task.failed_files;@@ -384,7 +406,7 @@ export function getEnhancedListDisappearedFilePaths(interface ProcessingFileOverlay {task_id: string;source_url: string;- status: "active" | "failed" | "processing" | "cancelled";+ status: "active" | "failed" | "processing" | "cancelled" | "skipped";error?: string;}diff --git a/frontend/tests/core/diverse-format-ingestion.spec.ts b/frontend/tests/core/diverse-format-ingestion.spec.tsindex 5c58727c6..e09d33ab8 100644--- a/frontend/tests/core/diverse-format-ingestion.spec.ts+++ b/frontend/tests/core/diverse-format-ingestion.spec.ts@@ -43,10 +43,12 @@ test.describe("Diverse Format Ingestion. @33219208 , @34581152 , @34581153 , @34}}- // Step 2: After all uploads complete, verify each file's status becomes Active+ // Step 2: After all uploads complete, verify each file was processed without+ // error. Active = newly indexed; Duplicate = correctly identified as already+ // present. Both are successful outcomes on this path.for (const uploadedFileName of uploadedFiles) {try {- await knowledge.verifyDocumentActive(uploadedFileName);+ await knowledge.verifyDocumentProcessed(uploadedFileName);} catch (error) {throw new Error(`Failed to verify ${uploadedFileName}: ${error}`);}diff --git a/frontend/tests/pages/Knowledge.ts b/frontend/tests/pages/Knowledge.tsindex 13abcec65..ded39a2ad 100644--- a/frontend/tests/pages/Knowledge.ts+++ b/frontend/tests/pages/Knowledge.ts@@ -754,6 +754,19 @@ export class Knowledge {await searchInp.clear();return;}+ // Terminal statuses will never transition to Active — fail immediately+ // instead of burning the full 5-minute deadline before throwing.+ const normalised = statusText.trim().toLowerCase();+ if (+ normalised === "failed" ||+ normalised === "cancelled" ||+ normalised === "duplicate"+ ) {+ await searchInp.clear();+ throw new Error(+ `Document "${docName}" reached terminal status "${statusText.trim()}" and will never become Active`,+ );+ }logger.info(` ⏳ Document "${docName}" status: "${statusText.trim()}", waiting...`,);@@ -765,6 +778,81 @@ export class Knowledge {);}+ /**+ * Verify that a document has been fully processed — either indexed as Active+ * or recognised as a content duplicate (Duplicate). Both outcomes mean the+ * system handled the file without error; use this instead of+ * verifyDocumentActive when duplicate detection is expected to fire.+ */+ async verifyDocumentProcessed(docName: string) {+ await this.open();+ const searchInp = this.searchInput();+ const searchReady = await searchInp+ .waitFor({ state: "visible", timeout: 15000 })+ .then(() => true)+ .catch(() => false);+ if (!searchReady) {+ throw new Error(+ `Knowledge grid search input not visible — page may not have loaded correctly`,+ );+ }++ const deadline = Date.now() + 300000;+ while (Date.now() < deadline) {+ await searchInp.clear();+ await searchInp.fill(docName);+ await this.page.waitForTimeout(800);+ let row = this.getRowById(docName).first();+ if (!(await row.isVisible().catch(() => false))) {+ const partial = this.getSourceCellByPartialText(docName);+ if (await partial.isVisible().catch(() => false)) {+ row = partial.locator('xpath=ancestor::*[@role="row"][1]');+ }+ }+ if (!(await row.isVisible().catch(() => false))) {+ logger.info(` ⏳ Document "${docName}" not yet visible, retrying...`);+ await searchInp.clear();+ continue;+ }+ await this.grid().evaluate((el) => {+ el.scrollLeft = el.scrollWidth;+ });+ await this.page.waitForTimeout(300);+ const status = this.getStatusCell(row);+ let statusText = (await status.innerText().catch(() => "")) || "";+ if (!statusText) {+ statusText = (await status.textContent().catch(() => "")) || "";+ }+ if (statusText.includes("Animated Processing Icon")) {+ statusText = "Processing";+ }+ const normalised = statusText.trim().toLowerCase();+ // Both Active and Duplicate are successful terminal outcomes.+ if (normalised.includes("active") || normalised === "duplicate") {+ await searchInp.clear();+ logger.info(+ ` ✅ Document "${docName}" processed with status "${statusText.trim()}"`,+ );+ return;+ }+ // Hard-failure terminals — throw immediately.+ if (normalised === "failed" || normalised === "cancelled") {+ await searchInp.clear();+ throw new Error(+ `Document "${docName}" reached terminal status "${statusText.trim()}"`,+ );+ }+ logger.info(+ ` ⏳ Document "${docName}" status: "${statusText.trim()}", waiting...`,+ );+ await this.page.waitForTimeout(5000);+ }+ await searchInp.clear();+ throw new Error(+ `Document "${docName}" did not reach a processed status within the timeout`,+ );+ }+/*** Verify that a document has been removed from the knowledge grid.* Searches for the document, and asserts it is NOT visible.diff --git a/src/models/processors.py b/src/models/processors.pyindex e1b4d41d3..99eb44f5f 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -31,6 +31,9 @@ DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"TEXT_PARSER_LABEL = "Text Parser"DUPLICATE_FILENAME_WARNING = "A file with this name already exists."+DUPLICATE_CONTENT_WARNING = (+ "Identical content already exists in the knowledge base under a different filename."+)if TYPE_CHECKING:from connectors.base import DocumentACL@@ -1608,6 +1611,38 @@ class LangflowFileProcessor(TaskProcessor):self.mark_duplicate_skipped(upload_task, file_task)return+ # Compute the content hash early — before reading file bytes into+ # memory and before submitting any work to Docling so the duplicate+ # guard fires as cheaply as possible.+ #+ # The guard only runs on the "proceed" path (no filename match).+ # When duplicate_action == "replaced" the caller asked to replace an+ # existing same-name document: the old chunks were already deleted+ # above, so we must not short-circuit here even if the hash matches+ # (that would leave the index empty after the delete).+ file_hash = hash_id(item)+ file_task.document_id = file_hash++ if duplicate_action == "proceed" and await self.check_document_exists(+ file_hash, opensearch_client+ ):+ # Identical content is already indexed under a different filename.+ # Report it as a warning rather than silently overwriting.+ # Mirrors the connector path (processors.py:1212) but uses a+ # distinct reason so the UI can distinguish it from a filename+ # collision.+ file_task.status = TaskStatus.SKIPPED+ file_task.error = None+ file_task.result = {+ "status": "skipped",+ "reason": "duplicate_content",+ "warning": DUPLICATE_CONTENT_WARNING,+ "document_id": file_hash,+ }+ file_task.updated_at = time.time()+ upload_task.successful_files += 1+ return+# Read file content for processingwith open(item, "rb") as f:content = f.read()@@ -1634,9 +1669,6 @@ class LangflowFileProcessor(TaskProcessor):# Prepare metadata tweaks similar to API endpointfinal_tweaks = self.tweaks.copy() if self.tweaks else {}- file_hash = hash_id(item)- file_task.document_id = file_hash-# Build settings with fresh OCR/pictureDescriptions from live# config so retries pick up configuration changes.config = get_openrag_config()diff --git a/tests/unit/known_failures.txt b/tests/unit/known_failures.txtindex 413d5589c..84f412fb9 100644--- a/tests/unit/known_failures.txt+++ b/tests/unit/known_failures.txt@@ -35,3 +35,5 @@ tests/unit/test_settings_provider_removal_defaults.py::TestFirstConfiguredEmbeddtests/unit/test_settings_provider_removal_defaults.py::TestProviderRemovalLlmDefault::test_remove_openai_falls_back_to_ollama_empty_modeltests/unit/test_settings_refresh_endpoint.py::test_refresh_endpoint_requires_auth_in_auth_modetests/unit/test_settings_refresh_endpoint.py::test_refresh_endpoint_returns_expected_http_response_shape+tests/unit/test_document_index_writer_bulk_retry.py::test_odd_length_bulk_body_raises+tests/unit/test_processor_mapping_client.py::test_standard_processor_uses_shared_writer_for_embedding_mapping_and_writesdiff --git a/tests/unit/test_langflow_processor_content_dedupe.py b/tests/unit/test_langflow_processor_content_dedupe.pynew file mode 100644index 000000000..3adb50a0c--- /dev/null+++ b/tests/unit/test_langflow_processor_content_dedupe.py@@ -0,0 +1,243 @@+"""Regression tests for content-duplicate detection in LangflowFileProcessor.++When a user uploads a byte-identical file under a different name, the filename+guard (resolve_duplicate_filename) does not fire — the names differ. The+processor must fall through to a content-hash check (check_document_exists)+and surface the result as SKIPPED with reason=duplicate_content, mirroring the+behaviour of the connector and traditional paths.++Relates to issue #2388.+"""++from unittest.mock import AsyncMock, MagicMock++import pytest++from models.processors import DUPLICATE_CONTENT_WARNING, LangflowFileProcessor+from models.tasks import FileTask, TaskStatus, UploadTask++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _build_processor() -> LangflowFileProcessor:+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=AsyncMock())++ langflow_file_service = MagicMock()+ langflow_file_service.upload_and_ingest_file = AsyncMock(+ return_value={"status": "indexed", "id": "hash-1"}+ )++ return LangflowFileProcessor(+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ owner_user_id="user-1",+ jwt_token="Bearer token",+ )+++# ---------------------------------------------------------------------------+# Tests+# ---------------------------------------------------------------------------+++async def test_langflow_processor_skips_on_content_duplicate(tmp_path):+ """Re-uploading identical content under a new filename must be SKIPPED with+ a user-visible warning, not silently overwritten."""+ processor = _build_processor()++ # Filename check: no existing document with this name → proceed.+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")+ # Content check: identical bytes already in the index.+ processor.check_document_exists = AsyncMock(return_value=True)++ item = tmp_path / "leave-policy-COPY.pdf"+ item.write_bytes(b"%PDF-1.4 identical content")++ file_task = FileTask(file_path=str(item))+ file_task.filename = "leave-policy-COPY.pdf"+ upload_task = UploadTask(task_id="task-1", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ assert file_task.status == TaskStatus.SKIPPED+ assert file_task.error is None+ assert file_task.result == {+ "status": "skipped",+ "reason": "duplicate_content",+ "warning": DUPLICATE_CONTENT_WARNING,+ "document_id": file_task.document_id,+ }+ assert upload_task.failed_files == 0+ assert upload_task.successful_files == 1+ # Expensive work (Docling + Langflow) must not run for a content duplicate.+ processor.langflow_file_service.upload_and_ingest_file.assert_not_called()+++async def test_langflow_processor_content_check_uses_file_hash(tmp_path):+ """check_document_exists must be called with the content hash of the file,+ so two identical files produce the same hash regardless of their names."""+ from utils.hash_utils import hash_id++ processor = _build_processor()+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")+ processor.check_document_exists = AsyncMock(return_value=True)++ content = b"%PDF-1.4 hash-me"+ item = tmp_path / "copy.pdf"+ item.write_bytes(content)++ file_task = FileTask(file_path=str(item))+ file_task.filename = "copy.pdf"+ upload_task = UploadTask(task_id="task-2", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ expected_hash = hash_id(str(item))+ processor.check_document_exists.assert_awaited_once()+ actual_hash_arg = processor.check_document_exists.await_args.args[0]+ assert actual_hash_arg == expected_hash+ assert file_task.document_id == expected_hash+++async def test_langflow_processor_new_content_proceeds(tmp_path):+ """When no content duplicate exists, processing must continue normally —+ the guard must be transparent for new files."""+ processor = _build_processor()+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")+ # Content not yet indexed.+ processor.check_document_exists = AsyncMock(return_value=False)+ # Post-ingest verification: file visible after ingest.+ processor.check_filename_exists = AsyncMock(return_value=True)++ item = tmp_path / "new-file.pdf"+ item.write_bytes(b"%PDF-1.4 brand new")++ file_task = FileTask(file_path=str(item))+ file_task.filename = "new-file.pdf"+ upload_task = UploadTask(task_id="task-3", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ assert file_task.status == TaskStatus.COMPLETED+ assert upload_task.successful_files == 1+ processor.langflow_file_service.upload_and_ingest_file.assert_awaited_once()+++async def test_langflow_processor_filename_duplicate_still_wins(tmp_path):+ """The filename guard fires first; if it skips, the content guard is never+ reached (and upload_and_ingest_file is still not called)."""+ processor = _build_processor()+ # Filename guard skips.+ processor.resolve_duplicate_filename = AsyncMock(return_value="skip")+ processor.check_document_exists = AsyncMock(return_value=True)++ item = tmp_path / "report.pdf"+ item.write_bytes(b"%PDF-1.4 something")++ file_task = FileTask(file_path=str(item))+ file_task.filename = "report.pdf"+ upload_task = UploadTask(task_id="task-4", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ assert file_task.status == TaskStatus.SKIPPED+ assert file_task.result["reason"] == "duplicate_filename"+ # Content check must never run when the filename guard already skipped.+ processor.check_document_exists.assert_not_called()+ processor.langflow_file_service.upload_and_ingest_file.assert_not_called()+++async def test_langflow_processor_replace_same_name_same_content_proceeds(tmp_path):+ """replace_duplicates=True with a same-name file whose content hash already+ exists must NOT trigger the content-duplicate skip. The old chunks were+ already deleted by resolve_duplicate_filename; bailing out here would leave+ the index empty. Ingest must proceed to re-index the document."""+ processor = _build_processor()+ # Filename guard deleted the old chunks and returned "replaced".+ processor.resolve_duplicate_filename = AsyncMock(return_value="replaced")+ # Hash matches the just-deleted document — must not short-circuit.+ processor.check_document_exists = AsyncMock(return_value=True)+ # Post-ingest verification: file visible after ingest.+ processor.check_filename_exists = AsyncMock(return_value=True)++ item = tmp_path / "report.pdf"+ item.write_bytes(b"%PDF-1.4 same bytes")++ file_task = FileTask(file_path=str(item))+ file_task.filename = "report.pdf"+ upload_task = UploadTask(task_id="task-5", total_files=1)++ await processor.process_item(upload_task, str(item), file_task)++ # Content guard must not have fired — ingest must have run.+ processor.langflow_file_service.upload_and_ingest_file.assert_awaited_once()+ assert file_task.status == TaskStatus.COMPLETED+ assert upload_task.successful_files == 1+ assert upload_task.failed_files == 0+++async def test_langflow_processor_concurrent_same_hash_only_one_proceeds(tmp_path):+ """Two concurrent uploads with identical content must result in exactly one+ Docling/Langflow submission. The second coroutine sees the hash already+ present (as it would after the first ingest completes) and skips.++ This is a best-effort in-process guard: check_document_exists is called+ before upload_and_ingest_file, so the race window is limited to the gap+ between the check and the write. A full atomic claim would require a+ distributed lock (tracked separately); this test verifies the guard position+ is correct and that the skip path is exercised under concurrency."""+ import asyncio++ call_count = 0++ async def fake_upload(**kwargs):+ nonlocal call_count+ call_count += 1+ return {"status": "indexed", "id": "hash-concurrent"}++ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=AsyncMock())++ langflow_file_service = MagicMock()+ langflow_file_service.upload_and_ingest_file = fake_upload++ item = tmp_path / "concurrent.pdf"+ item.write_bytes(b"%PDF-1.4 concurrent content")++ # Simulate: first call sees hash absent, second sees it present (post-ingest).+ exists_results = [False, True]++ async def fake_check_document_exists(file_hash, client):+ return exists_results.pop(0)++ async def run_one():+ processor = LangflowFileProcessor(+ langflow_file_service=langflow_file_service,+ session_manager=session_manager,+ owner_user_id="user-1",+ jwt_token="Bearer token",+ )+ processor.resolve_duplicate_filename = AsyncMock(return_value="proceed")+ processor.check_document_exists = fake_check_document_exists+ processor.check_filename_exists = AsyncMock(return_value=True)+ ft = FileTask(file_path=str(item))+ ft.filename = "concurrent.pdf"+ ut = UploadTask(task_id="task-concurrent", total_files=2)+ await processor.process_item(ut, str(item), ft)+ return ft++ results = await asyncio.gather(run_one(), run_one())++ statuses = {r.status for r in results}+ assert call_count == 1, "upload_and_ingest_file must be called exactly once"+ assert TaskStatus.COMPLETED in statuses+ assert TaskStatus.SKIPPED in statuses