Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix(frontend): show unsupported-file ingest warning and reuse overwrite flow for folder duplicates

Olfa Maslahdb1c468a10c6
diff --git a/frontend/components/duplicate-handling-dialog.tsx b/frontend/components/duplicate-handling-dialog.tsx
index 064e092f3..1d19cf357 100644
--- a/frontend/components/duplicate-handling-dialog.tsx
+++ b/frontend/components/duplicate-handling-dialog.tsx
@@ -17,24 +17,47 @@ interface DuplicateHandlingDialogProps {
onOpenChange: (open: boolean) => void;
onOverwrite: () => void | Promise<void>;
isLoading?: boolean;
+ duplicateLabel?: string;
+ duplicateCount?: number;
}
export const DuplicateHandlingDialog: React.FC<
DuplicateHandlingDialogProps
-> = ({ open, onOpenChange, onOverwrite, isLoading = false }) => {
+> = ({
+ open,
+ onOpenChange,
+ onOverwrite,
+ isLoading = false,
+ duplicateLabel,
+ duplicateCount,
+}) => {
const handleOverwrite = async () => {
await onOverwrite();
onOpenChange(false);
};
+ const description =
+ typeof duplicateCount === "number"
+ ? duplicateCount === 1
+ ? "1 duplicate document already exists. Overwriting will replace the existing document version. This can't be undone."
+ : `${duplicateCount} duplicate documents already exist. Overwriting will replace the existing document versions. This can't be undone.`
+ : duplicateLabel
+ ? `A document with "${duplicateLabel}" already exists. Overwriting will replace the existing document version. This can't be undone.`
+ : "Overwriting will replace the existing document with another version. This can't be undone.";
+ const overwriteLabel =
+ typeof duplicateCount === "number" ? "Overwrite duplicates" : "Overwrite";
+ const cancelLabel =
+ typeof duplicateCount === "number"
+ ? "Skip duplicates & continue"
+ : "Cancel";
+
return (
<Dialog open={open} onOpenChange={onOpenChange}>
- <DialogContent className="sm:max-w-[450px]">
+ <DialogContent className="sm:max-w-[520px]">
<DialogHeader>
<DialogTitle>Overwrite document</DialogTitle>
<DialogDescription className="pt-2 text-muted-foreground">
- Overwriting will replace the existing document with another version.
- This can&apos;t be undone.
+ {description}
</DialogDescription>
</DialogHeader>
@@ -45,8 +68,9 @@ export const DuplicateHandlingDialog: React.FC<
onClick={() => onOpenChange(false)}
disabled={isLoading}
size="sm"
+ className="whitespace-nowrap"
>
- Cancel
+ {cancelLabel}
</Button>
<Button
type="button"
@@ -54,10 +78,10 @@ export const DuplicateHandlingDialog: React.FC<
size="sm"
onClick={handleOverwrite}
disabled={isLoading}
- className="flex items-center gap-2 !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"
+ className="flex items-center gap-2 whitespace-nowrap !bg-accent-amber-foreground hover:!bg-foreground text-primary-foreground"
>
<RotateCcw className="h-3.5 w-3.5" />
- Overwrite
+ {overwriteLabel}
</Button>
</DialogFooter>
</DialogContent>
diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx
index 0895f9ba1..bf72c7408 100644
--- a/frontend/components/knowledge-dropdown.tsx
+++ b/frontend/components/knowledge-dropdown.tsx
@@ -73,6 +73,37 @@ export const SUPPORTED_FILE_TYPES = {
export const SUPPORTED_EXTENSIONS = Object.values(SUPPORTED_FILE_TYPES).flat();
+const getFilenameVariants = (filename: string): string[] => {
+ const dotIndex = filename.lastIndexOf(".");
+ if (dotIndex === -1) return [filename];
+
+ const baseName = filename.slice(0, dotIndex);
+ const extension = filename.slice(dotIndex).toLowerCase();
+
+ if (extension === ".txt") return [filename, `${baseName}.md`];
+ if (extension === ".md") return [filename, `${baseName}.txt`];
+
+ return [filename];
+};
+
+const isDuplicateFile = async (file: File): Promise<boolean> => {
+ const variants = getFilenameVariants(file.name);
+ const checks = await Promise.all(
+ variants.map(async (variantName) => {
+ const variantFile =
+ variantName === file.name
+ ? file
+ : new File([file], variantName, {
+ type: file.type,
+ lastModified: file.lastModified,
+ });
+ const checkData = await duplicateCheck(variantFile);
+ return checkData.exists;
+ }),
+ );
+ return checks.some(Boolean);
+};
+
const FileIconWithColor = ({ className }: { className?: string }) => (
<FileIcon className={cn(className, "text-muted-foreground")} />
);
@@ -100,6 +131,16 @@ export function KnowledgeDropdown() {
const [s3Configured, setS3Configured] = useState(false);
const [pendingFile, setPendingFile] = useState<File | null>(null);
const [duplicateFilename, setDuplicateFilename] = useState<string>("");
+ const [duplicateCountForDialog, setDuplicateCountForDialog] = useState<
+ number | undefined
+ >(undefined);
+ const [pendingFolderUpload, setPendingFolderUpload] = useState<{
+ allFiles: File[];
+ nonDuplicateFiles: File[];
+ duplicateCount: number;
+ unsupportedCount: number;
+ } | null>(null);
+ const isFolderOverwriteConfirmedRef = useRef(false);
const [cloudConnectors, setCloudConnectors] = useState<{
[key: string]: {
name: string;
@@ -249,13 +290,13 @@ export function KnowledgeDropdown() {
try {
console.log("[Duplicate Check] Checking file:", file.name);
- const checkData = await duplicateCheck(file);
- console.log("[Duplicate Check] Result:", checkData);
+ const exists = await isDuplicateFile(file);
- if (checkData.exists) {
+ if (exists) {
console.log("[Duplicate Check] Duplicate detected, showing dialog");
setPendingFile(file);
setDuplicateFilename(file.name);
+ setDuplicateCountForDialog(undefined);
setShowDuplicateDialog(true);
resetFileInput();
return;
@@ -299,7 +340,54 @@ export function KnowledgeDropdown() {
}
};
+ const uploadFolderBatches = async (
+ filesToUpload: File[],
+ replace: boolean,
+ ) => {
+ const batches: File[][] = [];
+ for (let i = 0; i < filesToUpload.length; i += uploadBatchSize) {
+ batches.push(filesToUpload.slice(i, i + uploadBatchSize));
+ }
+
+ console.log(
+ `[Folder Upload] Uploading ${filesToUpload.length} file(s) in ${batches.length} batch(es), replace=${replace}`,
+ );
+
+ for (const batch of batches) {
+ try {
+ const result = await uploadFiles(batch, replace);
+ addTask(result.taskId);
+ console.log(
+ `[Folder Upload] Batch uploaded: taskId=${result.taskId}, files=${result.fileCount}`,
+ );
+ } catch (error) {
+ console.error("[Folder Upload] Batch upload failed:", error);
+ toast.error("Batch upload failed", {
+ description: error instanceof Error ? error.message : "Unknown error",
+ });
+ }
+ }
+
+ refetchTasks();
+ };
+
const handleOverwriteFile = async () => {
+ if (pendingFolderUpload) {
+ isFolderOverwriteConfirmedRef.current = true;
+ const { allFiles, duplicateCount, unsupportedCount } =
+ pendingFolderUpload;
+ await uploadFolderBatches(allFiles, true);
+ const unsupportedMessage =
+ unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";
+ toast.success(
+ `Processed ${allFiles.length} file(s), including ${duplicateCount} overwrite(s)${unsupportedMessage}`,
+ );
+ setPendingFolderUpload(null);
+ setDuplicateFilename("");
+ setDuplicateCountForDialog(undefined);
+ return;
+ }
+
if (pendingFile) {
// Remove the old file from all search query caches before overwriting
queryClient.setQueriesData({ queryKey: ["search"] }, (oldData: []) => {
@@ -314,9 +402,46 @@ export function KnowledgeDropdown() {
setPendingFile(null);
setDuplicateFilename("");
+ setDuplicateCountForDialog(undefined);
}
};
+ const handleDuplicateDialogOpenChange = async (open: boolean) => {
+ if (!open && pendingFolderUpload) {
+ if (isFolderOverwriteConfirmedRef.current) {
+ isFolderOverwriteConfirmedRef.current = false;
+ } else {
+ const { nonDuplicateFiles, duplicateCount, unsupportedCount } =
+ pendingFolderUpload;
+ if (nonDuplicateFiles.length > 0) {
+ await uploadFolderBatches(nonDuplicateFiles, false);
+ const extraParts: string[] = [];
+ if (duplicateCount > 0) {
+ extraParts.push(`skipped ${duplicateCount} duplicate(s)`);
+ }
+ if (unsupportedCount > 0) {
+ extraParts.push(`skipped ${unsupportedCount} unsupported`);
+ }
+ const suffix =
+ extraParts.length > 0 ? `, ${extraParts.join(", ")}` : "";
+ toast.success(
+ `Processed ${nonDuplicateFiles.length} file(s)${suffix}`,
+ );
+ } else {
+ toast.info(
+ "Skipped duplicate files. All selected files were duplicates, so nothing was uploaded.",
+ );
+ }
+ }
+
+ setPendingFolderUpload(null);
+ setDuplicateFilename("");
+ setDuplicateCountForDialog(undefined);
+ }
+
+ setShowDuplicateDialog(open);
+ };
+
const handleFolderSelect = async (
event: React.ChangeEvent<HTMLInputElement>,
) => {
@@ -334,6 +459,7 @@ export function KnowledgeDropdown() {
.toLowerCase();
return SUPPORTED_EXTENSIONS.includes(ext);
});
+ const unsupportedCount = fileList.length - filteredFiles.length;
if (filteredFiles.length === 0) {
toast.error("No supported files found", {
@@ -359,8 +485,8 @@ export function KnowledgeDropdown() {
const duplicateResults = await Promise.all(
cleanFiles.map(async (file) => {
try {
- const checkData = await duplicateCheck(file);
- return { file, isDuplicate: checkData.exists };
+ const exists = await isDuplicateFile(file);
+ return { file, isDuplicate: exists };
} catch (error) {
console.error(
`[Folder Upload] Duplicate check failed for ${file.name}:`,
@@ -375,12 +501,33 @@ export function KnowledgeDropdown() {
const nonDuplicateFiles = duplicateResults
.filter((r) => !r.isDuplicate)
.map((r) => r.file);
- const skippedCount = duplicateResults.filter((r) => r.isDuplicate).length;
+ const duplicateCount = duplicateResults.filter(
+ (r) => r.isDuplicate,
+ ).length;
- if (skippedCount > 0) {
+ if (unsupportedCount > 0) {
+ toast.error(
+ `Unsupported files detected: only ${filteredFiles.length} of ${fileList.length} file(s) will be ingested.`,
+ {
+ description: `${unsupportedCount} file(s) have unsupported types and will be skipped.`,
+ },
+ );
+ }
+
+ if (duplicateCount > 0) {
console.log(
- `[Folder Upload] Skipping ${skippedCount} duplicate file(s)`,
+ `[Folder Upload] Found ${duplicateCount} duplicate file(s), showing overwrite dialog`,
);
+ setPendingFolderUpload({
+ allFiles: cleanFiles,
+ nonDuplicateFiles,
+ duplicateCount,
+ unsupportedCount,
+ });
+ setDuplicateFilename("");
+ setDuplicateCountForDialog(duplicateCount);
+ setShowDuplicateDialog(true);
+ return;
}
if (nonDuplicateFiles.length === 0) {
@@ -388,41 +535,12 @@ export function KnowledgeDropdown() {
return;
}
- // Chunk non-duplicate files into batches
- const batches: File[][] = [];
- for (let i = 0; i < nonDuplicateFiles.length; i += uploadBatchSize) {
- batches.push(nonDuplicateFiles.slice(i, i + uploadBatchSize));
- }
-
- console.log(
- `[Folder Upload] Uploading ${nonDuplicateFiles.length} file(s) in ${batches.length} batch(es)`,
+ await uploadFolderBatches(nonDuplicateFiles, false);
+ const unsupportedMessage =
+ unsupportedCount > 0 ? `, skipped ${unsupportedCount} unsupported` : "";
+ toast.success(
+ `Successfully processed ${nonDuplicateFiles.length} file(s)${unsupportedMessage}`,
);
-
- // Upload each batch as a single task
- for (const batch of batches) {
- try {
- const result = await uploadFiles(batch, false);
- addTask(result.taskId);
- console.log(
- `[Folder Upload] Batch uploaded: taskId=${result.taskId}, files=${result.fileCount}`,
- );
- } catch (error) {
- console.error("[Folder Upload] Batch upload failed:", error);
- toast.error("Batch upload failed", {
- description:
- error instanceof Error ? error.message : "Unknown error",
- });
- }
- }
-
- refetchTasks();
-
- const processedCount = nonDuplicateFiles.length;
- const message =
- skippedCount > 0
- ? `Processed ${processedCount} file(s), skipped ${skippedCount} duplicate(s)`
- : `Successfully processed ${processedCount} file(s)`;
- toast.success(message);
} catch (error) {
console.error("Folder upload error:", error);
toast.error("Folder upload failed", {
@@ -661,9 +779,11 @@ export function KnowledgeDropdown() {
{/* Duplicate Handling Dialog */}
<DuplicateHandlingDialog
open={showDuplicateDialog}
- onOpenChange={setShowDuplicateDialog}
+ onOpenChange={handleDuplicateDialogOpenChange}
onOverwrite={handleOverwriteFile}
- isLoading={fileUploading}
+ isLoading={fileUploading || folderLoading}
+ duplicateLabel={duplicateFilename}
+ duplicateCount={duplicateCountForDialog}
/>
</>
);