OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix(frontend): show unsupported-file ingest warning and reuse overwrite flow for folder duplicates
Olfa Maslahdb1c468a10c6
diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsxindex 064e092f3..1d19cf357 100644--- a/frontend/components/duplicate-handling-dialog.tsx+++ b/frontend/components/duplicate-handling-dialog.tsx@@ -17,24 +17,47 @@ interface DuplicateHandlingDialogProps {onOpenChange: (open: boolean) => void;onOverwrite: () => void | Promise<void>;isLoading?: boolean;+ duplicateLabel?: string;+ duplicateCount?: number;}export const DuplicateHandlingDialog: React.FC<DuplicateHandlingDialogProps-> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => {+> = ({+ open,+ onOpenChange,+ onOverwrite,+ isLoading = false,+ duplicateLabel,+ duplicateCount,+}) => {const handleOverwrite = async () => {await onOverwrite();onOpenChange(false);};+ const description =+ typeof duplicateCount === "number"+ ? duplicateCount === 1+ ? "1 duplicate document already exists. Overwriting will replace the existing document version. This can't be undone."+ : `${duplicateCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`+ : duplicateLabel+ ? `A document with "${duplicateLabel}" already exists. Overwriting will replace the existing document version. This can't be undone.`+ : "Overwriting will replace the existing document with another version. This can't be undone.";+ const overwriteLabel =+ typeof duplicateCount === "number" ? "Overwrite duplicates" : "Overwrite";+ const cancelLabel =+ typeof duplicateCount === "number"+ ? "Skip duplicates & continue"+ : "Cancel";+return (<Dialog open={open} onOpenChange={onOpenChange}>- <DialogContent className="sm:max-w-[450px]">+ <DialogContent className="sm:max-w-[520px]"><DialogHeader><DialogTitle>Overwrite document</DialogTitle><DialogDescription className="pt-2 text-muted-foreground">- Overwriting will replace the existing document with another version.- This can't be undone.+ {description}</DialogDescription></DialogHeader>@@ -45,8 +68,9 @@ export const DuplicateHandlingDialog: React.FC<onClick={() => onOpenChange(false)}disabled={isLoading}size="sm"+ className="whitespace-nowrap">- Cancel+ {cancelLabel}</Button><Buttontype="button"@@ -54,10 +78,10 @@ export const DuplicateHandlingDialog: React.FC<size="sm"onClick={handleOverwrite}disabled={isLoading}- className="flex items-center gap-2 !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"+ className="flex items-center gap-2 whitespace-nowrap !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"><RotateCcw className="h-3.5 w-3.5" />- Overwrite+ {overwriteLabel}</Button></DialogFooter></DialogContent>diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsxindex 0895f9ba1..bf72c7408 100644--- a/frontend/components/knowledge-dropdown.tsx+++ b/frontend/components/knowledge-dropdown.tsx@@ -73,6 +73,37 @@ export const SUPPORTED_FILE_TYPES = {export const SUPPORTED_EXTENSIONS = Object.values(SUPPORTED_FILE_TYPES).flat();+const getFilenameVariants = (filename: string): string[] => {+ const dotIndex = filename.lastIndexOf(".");+ if (dotIndex === -1) return [filename];++ const baseName = filename.slice(0, dotIndex);+ const extension = filename.slice(dotIndex).toLowerCase();++ if (extension === ".txt") return [filename, `${baseName}.md`];+ if (extension === ".md") return [filename, `${baseName}.txt`];++ return [filename];+};++const isDuplicateFile = async (file: File): Promise<boolean> => {+ const variants = getFilenameVariants(file.name);+ const checks = await Promise.all(+ variants.map(async (variantName) => {+ const variantFile =+ variantName === file.name+ ? file+ : new File([file], variantName, {+ type: file.type,+ lastModified: file.lastModified,+ });+ const checkData = await duplicateCheck(variantFile);+ return checkData.exists;+ }),+ );+ return checks.some(Boolean);+};+const FileIconWithColor = ({ className }: { className?: string }) => (<FileIcon className={cn(className, "text-muted-foreground")} />);@@ -100,6 +131,16 @@ export function KnowledgeDropdown() {const [s3Configured, setS3Configured] = useState(false);const [pendingFile, setPendingFile] = useState<File | null>(null);const [duplicateFilename, setDuplicateFilename] = useState<string>("");+ const [duplicateCountForDialog, setDuplicateCountForDialog] = useState<+ number | undefined+ >(undefined);+ const [pendingFolderUpload, setPendingFolderUpload] = useState<{+ allFiles: File[];+ nonDuplicateFiles: File[];+ duplicateCount: number;+ unsupportedCount: number;+ } | null>(null);+ const isFolderOverwriteConfirmedRef = useRef(false);const [cloudConnectors, setCloudConnectors] = useState<{[key: string]: {name: string;@@ -249,13 +290,13 @@ export function KnowledgeDropdown() {try {console.log("[Duplicate Check] Checking file:", file.name);- const checkData = await duplicateCheck(file);- console.log("[Duplicate Check] Result:", checkData);+ const exists = await isDuplicateFile(file);- if (checkData.exists) {+ if (exists) {console.log("[Duplicate Check] Duplicate detected, showing dialog");setPendingFile(file);setDuplicateFilename(file.name);+ setDuplicateCountForDialog(undefined);setShowDuplicateDialog(true);resetFileInput();return;@@ -299,7 +340,54 @@ export function KnowledgeDropdown() {}};+ const uploadFolderBatches = async (+ filesToUpload: File[],+ replace: boolean,+ ) => {+ const batches: File[][] = [];+ for (let i = 0; i < filesToUpload.length; i += uploadBatchSize) {+ batches.push(filesToUpload.slice(i, i + uploadBatchSize));+ }++ console.log(+ `[Folder Upload] Uploading ${filesToUpload.length} file(s) in ${batches.length} batch(es), replace=${replace}`,+ );++ for (const batch of batches) {+ try {+ const result = await uploadFiles(batch, replace);+ addTask(result.taskId);+ console.log(+ `[Folder Upload] Batch uploaded: taskId=${result.taskId}, files=${result.fileCount}`,+ );+ } catch (error) {+ console.error("[Folder Upload] Batch upload failed:", error);+ toast.error("Batch upload failed", {+ description: error instanceof Error ? error.message : "Unknown error",+ });+ }+ }++ refetchTasks();+ };+const handleOverwriteFile = async () => {+ if (pendingFolderUpload) {+ isFolderOverwriteConfirmedRef.current = true;+ const { allFiles, duplicateCount, unsupportedCount } =+ pendingFolderUpload;+ await uploadFolderBatches(allFiles, true);+ const unsupportedMessage =+ unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";+ toast.success(+ `Processed ${allFiles.length} file(s), including ${duplicateCount} overwrite(s)${unsupportedMessage}`,+ );+ setPendingFolderUpload(null);+ setDuplicateFilename("");+ setDuplicateCountForDialog(undefined);+ return;+ }+if (pendingFile) {// Remove the old file from all search query caches before overwritingqueryClient.setQueriesData({ queryKey: ["search"] }, (oldData: []) => {@@ -314,9 +402,46 @@ export function KnowledgeDropdown() {setPendingFile(null);setDuplicateFilename("");+ setDuplicateCountForDialog(undefined);}};+ const handleDuplicateDialogOpenChange = async (open: boolean) => {+ if (!open && pendingFolderUpload) {+ if (isFolderOverwriteConfirmedRef.current) {+ isFolderOverwriteConfirmedRef.current = false;+ } else {+ const { nonDuplicateFiles, duplicateCount, unsupportedCount } =+ pendingFolderUpload;+ if (nonDuplicateFiles.length > 0) {+ await uploadFolderBatches(nonDuplicateFiles, false);+ const extraParts: string[] = [];+ if (duplicateCount > 0) {+ extraParts.push(`skipped ${duplicateCount} duplicate(s)`);+ }+ if (unsupportedCount > 0) {+ extraParts.push(`skipped ${unsupportedCount} unsupported`);+ }+ const suffix =+ extraParts.length > 0 ? `, ${extraParts.join(", ")}` : "";+ toast.success(+ `Processed ${nonDuplicateFiles.length} file(s)${suffix}`,+ );+ } else {+ toast.info(+ "Skipped duplicate files. All selected files were duplicates, so nothing was uploaded.",+ );+ }+ }++ setPendingFolderUpload(null);+ setDuplicateFilename("");+ setDuplicateCountForDialog(undefined);+ }++ setShowDuplicateDialog(open);+ };+const handleFolderSelect = async (event: React.ChangeEvent<HTMLInputElement>,) => {@@ -334,6 +459,7 @@ export function KnowledgeDropdown() {.toLowerCase();return SUPPORTED_EXTENSIONS.includes(ext);});+ const unsupportedCount = fileList.length - filteredFiles.length;if (filteredFiles.length === 0) {toast.error("No supported files found", {@@ -359,8 +485,8 @@ export function KnowledgeDropdown() {const duplicateResults = await Promise.all(cleanFiles.map(async (file) => {try {- const checkData = await duplicateCheck(file);- return { file, isDuplicate: checkData.exists };+ const exists = await isDuplicateFile(file);+ return { file, isDuplicate: exists };} catch (error) {console.error(`[Folder Upload] Duplicate check failed for ${file.name}:`,@@ -375,12 +501,33 @@ export function KnowledgeDropdown() {const nonDuplicateFiles = duplicateResults.filter((r) => !r.isDuplicate).map((r) => r.file);- const skippedCount = duplicateResults.filter((r) => r.isDuplicate).length;+ const duplicateCount = duplicateResults.filter(+ (r) => r.isDuplicate,+ ).length;- if (skippedCount > 0) {+ if (unsupportedCount > 0) {+ toast.error(+ `Unsupported files detected: only ${filteredFiles.length} of ${fileList.length} file(s) will be ingested.`,+ {+ description: `${unsupportedCount} file(s) have unsupported types and will be skipped.`,+ },+ );+ }++ if (duplicateCount > 0) {console.log(- `[Folder Upload] Skipping ${skippedCount} duplicate file(s)`,+ `[Folder Upload] Found ${duplicateCount} duplicate file(s), showing overwrite dialog`,);+ setPendingFolderUpload({+ allFiles: cleanFiles,+ nonDuplicateFiles,+ duplicateCount,+ unsupportedCount,+ });+ setDuplicateFilename("");+ setDuplicateCountForDialog(duplicateCount);+ setShowDuplicateDialog(true);+ return;}if (nonDuplicateFiles.length === 0) {@@ -388,41 +535,12 @@ export function KnowledgeDropdown() {return;}- // Chunk non-duplicate files into batches- const batches: File[][] = [];- for (let i = 0; i < nonDuplicateFiles.length; i += uploadBatchSize) {- batches.push(nonDuplicateFiles.slice(i, i + uploadBatchSize));- }-- console.log(- `[Folder Upload] Uploading ${nonDuplicateFiles.length} file(s) in ${batches.length} batch(es)`,+ await uploadFolderBatches(nonDuplicateFiles, false);+ const unsupportedMessage =+ unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";+ toast.success(+ `Successfully processed ${nonDuplicateFiles.length} file(s)${unsupportedMessage}`,);-- // Upload each batch as a single task- for (const batch of batches) {- try {- const result = await uploadFiles(batch, false);- addTask(result.taskId);- console.log(- `[Folder Upload] Batch uploaded: taskId=${result.taskId}, files=${result.fileCount}`,- );- } catch (error) {- console.error("[Folder Upload] Batch upload failed:", error);- toast.error("Batch upload failed", {- description:- error instanceof Error ? error.message : "Unknown error",- });- }- }-- refetchTasks();-- const processedCount = nonDuplicateFiles.length;- const message =- skippedCount > 0- ? `Processed ${processedCount} file(s), skipped ${skippedCount} duplicate(s)`- : `Successfully processed ${processedCount} file(s)`;- toast.success(message);} catch (error) {console.error("Folder upload error:", error);toast.error("Folder upload failed", {@@ -661,9 +779,11 @@ export function KnowledgeDropdown() {{/* Duplicate Handling Dialog */}<DuplicateHandlingDialogopen={showDuplicateDialog}- onOpenChange={setShowDuplicateDialog}+ onOpenChange={handleDuplicateDialogOpenChange}onOverwrite={handleOverwriteFile}- isLoading={fileUploading}+ isLoading={fileUploading || folderLoading}+ duplicateLabel={duplicateFilename}+ duplicateCount={duplicateCountForDialog}/></>);