Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧺 fix: Skip Oversized and Duplicate Files Instead of the Whole Batch (#15358)

Danny Avila240e9e920f5e
diff --git a/client/src/hooks/Files/__tests__/useFileHandling.test.ts b/client/src/hooks/Files/__tests__/useFileHandling.test.ts
index 33738f7375..251947cf1f 100644
--- a/client/src/hooks/Files/__tests__/useFileHandling.test.ts
+++ b/client/src/hooks/Files/__tests__/useFileHandling.test.ts
@@ -154,11 +154,14 @@ jest.mock('../useUpdateFiles', () => ({
}));
jest.mock('~/utils', () => {
- const { validateFileSizes, validateFileDuplicates } = jest.requireActual('~/utils/files');
+ const { partitionUploads, validateFileSizes, validateFileLimit, validateFileDuplicates } =
+ jest.requireActual('~/utils/files');
return {
logger: { log: jest.fn() },
validateFiles: jest.fn(() => true),
+ partitionUploads: jest.fn(partitionUploads),
validateFileSizes: jest.fn(validateFileSizes),
+ validateFileLimit: jest.fn(validateFileLimit),
validateFileDuplicates: jest.fn(validateFileDuplicates),
cachePreview: jest.fn(),
getCachedPreview: jest.fn(() => undefined),
@@ -167,14 +170,18 @@ jest.mock('~/utils', () => {
});
const mockValidateFiles = jest.requireMock('~/utils').validateFiles;
+const mockPartitionUploads = jest.requireMock('~/utils').partitionUploads;
const mockValidateFileSizes = jest.requireMock('~/utils').validateFileSizes;
+const mockValidateFileLimit = jest.requireMock('~/utils').validateFileLimit;
const mockValidateFileDuplicates = jest.requireMock('~/utils').validateFileDuplicates;
describe('useFileHandling', () => {
beforeEach(() => {
jest.clearAllMocks();
mockValidateFiles.mockImplementation(() => true);
+ mockPartitionUploads.mockImplementation(jest.requireActual('~/utils/files').partitionUploads);
mockValidateFileSizes.mockImplementation(jest.requireActual('~/utils/files').validateFileSizes);
+ mockValidateFileLimit.mockImplementation(jest.requireActual('~/utils/files').validateFileLimit);
mockValidateFileDuplicates.mockImplementation(
jest.requireActual('~/utils/files').validateFileDuplicates,
);
@@ -374,6 +381,269 @@ describe('useFileHandling', () => {
}
});
+ it('uploads the files under the limit when one of them is oversized', async () => {
+ jest.useFakeTimers();
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileSizeLimit: 20, totalSizeLimit: 500 } },
+ });
+ const batch = [
+ makeSizedFile('small.txt', 'text/plain', 1 * megabyte),
+ makeSizedFile('huge.txt', 'text/plain', 21 * megabyte),
+ makeSizedFile('medium.txt', 'text/plain', 5 * megabyte),
+ ];
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ let accepted: boolean | undefined;
+ await act(async () => {
+ accepted = await result.current.handleFiles(batch);
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(accepted).toBe(true);
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockLocalize).toHaveBeenCalledWith('com_error_files_skipped_size', {
+ 0: '20',
+ 1: 'huge.txt',
+ });
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'com_error_files_skipped_size',
+ status: 'error',
+ duration: 5000,
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('names every oversized file it skipped', async () => {
+ jest.useFakeTimers();
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileSizeLimit: 20, totalSizeLimit: 500 } },
+ });
+ const batch = [
+ makeSizedFile('huge.txt', 'text/plain', 21 * megabyte),
+ makeSizedFile('small.txt', 'text/plain', 1 * megabyte),
+ makeSizedFile('massive.txt', 'text/plain', 40 * megabyte),
+ ];
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ await act(async () => {
+ await result.current.handleFiles(batch);
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+ expect(mockLocalize).toHaveBeenCalledWith('com_error_files_skipped_size', {
+ 0: '20',
+ 1: 'huge.txt, massive.txt',
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('uploads the rest of a batch when one file duplicates an attachment', async () => {
+ jest.useFakeTimers({ doNotFake: ['queueMicrotask'] });
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileSizeLimit: 20, totalSizeLimit: 500 } },
+ });
+ const alreadyAttached = makeSizedFile('report.txt', 'text/plain', 1 * megabyte);
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ await act(async () => {
+ await result.current.handleFiles([alreadyAttached]);
+ await Promise.resolve();
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('report.txt', 'text/plain', 1 * megabyte),
+ makeSizedFile('notes.txt', 'text/plain', 2 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockLocalize).toHaveBeenCalledWith('com_error_files_skipped_dupe', {
+ 0: 'report.txt',
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('does not announce skipped files when the batch is rejected anyway', async () => {
+ jest.useFakeTimers({ doNotFake: ['queueMicrotask'] });
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileSizeLimit: 20, totalSizeLimit: 7 } },
+ });
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('attached.txt', 'text/plain', 1 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+
+ mockLocalize.mockClear();
+ /** The duplicate drops out, but what is left still busts the total, so nothing uploads. */
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('attached.txt', 'text/plain', 1 * megabyte),
+ makeSizedFile('one.txt', 'text/plain', 4 * megabyte),
+ makeSizedFile('two.txt', 'text/plain', 4 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+ expect(mockLocalize).not.toHaveBeenCalledWith(
+ 'com_error_files_skipped_dupe',
+ expect.anything(),
+ );
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'Total file size limit exceeded: 7 MB',
+ status: 'error',
+ duration: 5000,
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('does not let an oversized file spend the last file limit slot', async () => {
+ jest.useFakeTimers({ doNotFake: ['queueMicrotask'] });
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileLimit: 2, fileSizeLimit: 20, totalSizeLimit: 500 } },
+ });
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('attached.txt', 'text/plain', 1 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+
+ /** One slot is left, so counting the oversized file would reject the valid one with it. */
+ let accepted: boolean | undefined;
+ await act(async () => {
+ accepted = await result.current.handleFiles([
+ makeSizedFile('huge.txt', 'text/plain', 21 * megabyte),
+ makeSizedFile('valid.txt', 'text/plain', 2 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(accepted).toBe(true);
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockMutate.mock.calls[1][0].get('file').name).toBe('valid.txt');
+ /** The count sees the survivors, not the selection it was picked from. */
+ const [countedAgainst] = mockValidateFileLimit.mock.calls.at(-1) ?? [];
+ expect(countedAgainst?.fileList.map((file: File) => file.name)).toEqual(['valid.txt']);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('counts only the files it will upload against the file limit', async () => {
+ jest.useFakeTimers({ doNotFake: ['queueMicrotask'] });
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileLimit: 2, fileSizeLimit: 20, totalSizeLimit: 500 } },
+ });
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('report.txt', 'text/plain', 1 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+
+ mockValidateFiles.mockClear();
+ await act(async () => {
+ await result.current.handleFiles([
+ makeSizedFile('report.txt', 'text/plain', 1 * megabyte),
+ makeSizedFile('notes.txt', 'text/plain', 2 * megabyte),
+ ]);
+ await Promise.resolve();
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ /** The duplicate is dropped before the count check, so it never spends a slot. */
+ const [{ fileList: counted }] = mockValidateFiles.mock.calls[0];
+ expect(counted.map((file: File) => file.name)).toEqual(['notes.txt']);
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('still applies the total size limit to the files left after a skip', async () => {
+ jest.useFakeTimers();
+ try {
+ mockFileConfig = mergeFileConfig({
+ endpoints: { default: { fileSizeLimit: 10, totalSizeLimit: 7 } },
+ });
+ const batch = [
+ makeSizedFile('huge.txt', 'text/plain', 21 * megabyte),
+ makeSizedFile('one.txt', 'text/plain', 4 * megabyte),
+ makeSizedFile('two.txt', 'text/plain', 4 * megabyte),
+ ];
+ const useFileHandling = await loadHook();
+ const { result } = renderHook(() => useFileHandling());
+
+ let accepted: boolean | undefined;
+ await act(async () => {
+ accepted = await result.current.handleFiles(batch);
+ });
+ await act(async () => {
+ jest.advanceTimersByTime(250);
+ });
+
+ expect(accepted).toBe(false);
+ expect(mockMutate).not.toHaveBeenCalled();
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'Total file size limit exceeded: 7 MB',
+ status: 'error',
+ duration: 5000,
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
it('applies the total size limit to the complete transformed batch', async () => {
jest.useFakeTimers();
try {
@@ -509,7 +779,11 @@ describe('useFileHandling', () => {
jest.advanceTimersByTime(250);
});
- expect(mockValidateFileDuplicates).toHaveBeenCalledTimes(2);
+ /** The duplicate only emerges once resizing has run, so it is the processed batch that
+ * has to catch it — and with nothing left to upload the batch still reports the plain
+ * duplicate error rather than a per-file skip notice. */
+ const [lastPartition] = mockPartitionUploads.mock.calls.at(-1) ?? [];
+ expect(lastPartition?.fileList[0].size).toBe(4 * megabyte);
expect(mockMutate).toHaveBeenCalledTimes(1);
expect(mockShowToast).toHaveBeenCalledWith({
message: 'com_error_files_dupe',
@@ -1329,7 +1603,7 @@ describe('useFileHandling', () => {
});
rerender();
- mockValidateFileDuplicates.mockClear();
+ mockPartitionUploads.mockClear();
mockImageDecodes = true;
await act(async () => {
await result.current.handleFiles([pick()]);
@@ -1338,7 +1612,7 @@ describe('useFileHandling', () => {
/** A reservation the failed decode left behind is merged into the next batch's
* validation, so re-picking the same file reads as a duplicate and its size
* keeps counting against the composer's limits. */
- const [{ files: validatedAgainst }] = mockValidateFileDuplicates.mock.calls[0];
+ const [{ files: validatedAgainst }] = mockPartitionUploads.mock.calls[0];
expect(validatedAgainst.size).toBe(0);
expect(mockMutate).toHaveBeenCalledTimes(1);
});
diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.ts
index 3e83419d48..58ded69e44 100644
--- a/client/src/hooks/Files/useFileHandling.ts
+++ b/client/src/hooks/Files/useFileHandling.ts
@@ -5,6 +5,7 @@ import { useToastContext } from '@librechat/client';
import { useQueryClient } from '@tanstack/react-query';
import { useRecoilValue, useSetRecoilState } from 'recoil';
import {
+ megabyte,
QueryKeys,
Constants,
EToolResources,
@@ -13,14 +14,22 @@ import {
getEndpointFileConfig,
defaultAssistantsVersion,
} from 'librechat-data-provider';
-import type { EModelEndpoint, TEndpointsConfig, TError } from 'librechat-data-provider';
+import type {
+ TError,
+ EModelEndpoint,
+ TEndpointsConfig,
+ EndpointFileConfig,
+} from 'librechat-data-provider';
import type { TConversation } from 'librechat-data-provider';
+import type { SkippedUpload, UploadPartition } from '~/utils';
import type { ExtendedFile, FileSetter } from '~/common';
import {
logger,
validateFiles,
cachePreview,
+ partitionUploads,
validateFileSizes,
+ validateFileLimit,
getCachedPreview,
removePreviewEntry,
validateFileDuplicates,
@@ -158,6 +167,37 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
);
const isTemporary = useRecoilValue(store.isTemporary);
const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
+
+ /** Names the files left out of a batch that is otherwise still uploading. Callers report a batch
+ * rejected in full through the existing all-or-nothing errors instead. */
+ const reportSkippedUploads = (
+ skipped: SkippedUpload[],
+ endpointFileConfig: EndpointFileConfig,
+ ) => {
+ if (skipped.length === 0) {
+ return;
+ }
+ const duplicates: string[] = [];
+ const oversized: string[] = [];
+ for (const { file, reason } of skipped) {
+ if (reason === 'duplicate') {
+ duplicates.push(file.name);
+ continue;
+ }
+ oversized.push(file.name);
+ }
+ if (duplicates.length > 0) {
+ setError(localize('com_error_files_skipped_dupe', { 0: duplicates.join(', ') }));
+ }
+ if (oversized.length > 0) {
+ setError(
+ localize('com_error_files_skipped_size', {
+ 0: `${(endpointFileConfig.fileSizeLimit ?? 0) / megabyte}`,
+ 1: oversized.join(', '),
+ }),
+ );
+ }
+ };
const { addFile, replaceFile, updateFileById, deleteFileById } = useUpdateFiles(fileSetter);
const { isConfigPending, waitForConfig, resizeImageIfNeeded } = useClientResize();
@@ -475,17 +515,35 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
return withoutSource;
})();
+ /** Drop duplicates one by one rather than rejecting everything picked alongside them, and hand
+ * the survivors to `validateFiles`. Sizes wait for the partition below, once processing has
+ * settled each file's final bytes, which is also where the file count is finally applied: a
+ * file still headed for the discard pile must not spend a `fileLimit` slot. */
+ const selection = partitionUploads({
+ files: filesForValidation,
+ fileList,
+ endpointFileConfig,
+ skipSizeValidation: true,
+ });
+ /** Nothing survived, so the whole selection is rejected and the untouched list reports it
+ * through the usual checks in their usual order. */
+ const acceptedFileList =
+ selection.keptIndices.length > 0
+ ? selection.keptIndices.map((index) => fileList[index])
+ : fileList;
+
/* Validate files */
let filesAreValid: boolean;
try {
filesAreValid = validateFiles({
files: filesForValidation,
- fileList,
+ fileList: acceptedFileList,
setError,
fileConfig: currentFileConfig,
endpointFileConfig,
toolResource: _toolResource,
skipSizeValidation: true,
+ skipBatchRules: selection.keptIndices.length > 0,
});
} catch (error) {
console.error('file validation error', error);
@@ -500,7 +558,7 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
/* Process files */
const processedUploads: ProcessedUpload[] = [];
- for (const [fileIndex, originalFile] of fileList.entries()) {
+ for (const [fileIndex, originalFile] of acceptedFileList.entries()) {
const file_id =
fileIndex === 0 && uploadLifecycle?.fileId != null && uploadLifecycle.fileId !== ''
? uploadLifecycle.fileId
@@ -619,31 +677,69 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
}
}
+ const discardProcessedUpload = ({ extendedFile, preview }: ProcessedUpload) => {
+ deleteFileById(extendedFile.file_id);
+ removePreviewEntry(extendedFile.file_id);
+ URL.revokeObjectURL(preview);
+ };
+
const discardProcessedUploads = () => {
- for (const { extendedFile, preview } of processedUploads) {
- deleteFileById(extendedFile.file_id);
- removePreviewEntry(extendedFile.file_id);
- URL.revokeObjectURL(preview);
+ for (const upload of processedUploads) {
+ discardProcessedUpload(upload);
}
filesRef.current = existingFiles;
};
const processedFileList = processedUploads.map(({ extendedFile }) => extendedFile.file as File);
- let batchIsValid: boolean;
+ let batch: UploadPartition;
+ let acceptedUploads: ProcessedUpload[];
try {
- batchIsValid =
- validateFileDuplicates({
+ batch = partitionUploads({
+ files: filesForValidation,
+ fileList: processedFileList,
+ endpointFileConfig,
+ });
+ acceptedUploads = batch.keptIndices.map((index) => processedUploads[index]);
+ const acceptedFiles = acceptedUploads.map(({ extendedFile }) => extendedFile.file as File);
+ /** `fileLimit` and `totalSizeLimit` describe the batch rather than any one file, so both are
+ * applied to whatever survived the per-file partition above. */
+ const batchIsValid =
+ acceptedUploads.length > 0 &&
+ validateFileLimit({
files: filesForValidation,
- fileList: processedFileList,
+ fileList: acceptedFiles,
setError,
+ endpointFileConfig,
}) &&
validateFileSizes({
files: filesForValidation,
- fileList: processedFileList,
+ fileList: acceptedFiles,
setError,
endpointFileConfig,
});
+ if (!batchIsValid) {
+ /** Nothing is left to upload, so this is the pre-existing all-or-nothing rejection and it
+ * keeps reporting itself that way instead of as per-file skip notices. */
+ if (acceptedUploads.length === 0) {
+ const noDuplicates = validateFileDuplicates({
+ files: filesForValidation,
+ fileList: processedFileList,
+ setError,
+ });
+ if (noDuplicates) {
+ validateFileSizes({
+ files: filesForValidation,
+ fileList: processedFileList,
+ setError,
+ endpointFileConfig,
+ });
+ }
+ }
+ discardProcessedUploads();
+ setFilesLoading(false);
+ return false;
+ }
} catch (error) {
console.error('file validation error', error);
setError('com_error_files_validation');
@@ -651,14 +747,16 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
setFilesLoading(false);
return false;
}
- if (!batchIsValid) {
- discardProcessedUploads();
- setFilesLoading(false);
- return false;
+
+ for (const { index } of batch.skipped) {
+ discardProcessedUpload(processedUploads[index]);
}
+ /** Held until the batch is known to be uploading: a selection that is rejected in full reports
+ * itself through the batch-level errors alone, never alongside a partial-success notice. */
+ reportSkippedUploads([...selection.skipped, ...batch.skipped], endpointFileConfig);
const filesWithProcessedUploads = new Map(existingFiles);
- for (const { extendedFile } of processedUploads) {
+ for (const { extendedFile } of acceptedUploads) {
filesWithProcessedUploads.set(extendedFile.file_id, extendedFile);
if (tracksReservations) {
uploadScope.recent.set(extendedFile.file_id, extendedFile);
@@ -666,7 +764,7 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
}
filesRef.current = filesWithProcessedUploads;
- for (const { extendedFile, preview, resizeDetails } of processedUploads) {
+ for (const { extendedFile, preview, resizeDetails } of acceptedUploads) {
if (resizeDetails) {
const { originalSize, newSize, compressionRatio } = resizeDetails;
showToast({
@@ -688,7 +786,7 @@ const useFileHandlingCore = (params: UseFileHandling | undefined, fileState: Fil
await startUpload(extendedFile, uploadLifecycle);
}
- return processedUploads.length > 0;
+ return acceptedUploads.length > 0;
};
const handleFiles = async (
diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json
index 5fd1b363f1..4b5776cbfe 100644
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@@ -379,6 +379,8 @@
"com_error_files_dupe": "Duplicate file detected.",
"com_error_files_empty": "Empty files are not allowed.",
"com_error_files_process": "An error occurred while processing the file.",
+ "com_error_files_skipped_dupe": "Skipped duplicate file(s): {{0}}",
+ "com_error_files_skipped_size": "Skipped file(s) over the {{0}} MB size limit: {{1}}",
"com_error_files_unsupported": "This file type can't be attached here.",
"com_error_files_upload": "An error occurred while uploading the file.",
"com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.",
diff --git a/client/src/utils/__tests__/validateFiles.spec.ts b/client/src/utils/__tests__/validateFiles.spec.ts
index a2a5d9f4f3..ffced653af 100644
--- a/client/src/utils/__tests__/validateFiles.spec.ts
+++ b/client/src/utils/__tests__/validateFiles.spec.ts
@@ -1,7 +1,7 @@
import { megabyte, fileConfig as defaultFileConfig } from 'librechat-data-provider';
import type { EndpointFileConfig, FileConfig } from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
-import { validateFiles, validateFileSizes } from '../files';
+import { validateFiles, validateFileSizes, partitionUploads } from '../files';
const supportedMimeTypes = defaultFileConfig.endpoints.default.supportedMimeTypes;
@@ -21,6 +21,13 @@ function makeFile(name: string, type: string, size: number): File {
return new File([content], name, { type });
}
+/** Stands in for a file of any size without allocating its bytes, which only the size rules read. */
+function makeSizedFile(name: string, type: string, size: number): File {
+ const file = new File(['content'], name, { type });
+ Object.defineProperty(file, 'size', { value: size });
+ return file;
+}
+
function makeExtendedFile(overrides: Partial<ExtendedFile> = {}): ExtendedFile {
return {
file_id: 'ext-1',
@@ -244,3 +251,117 @@ describe('validateFiles', () => {
expect(setError).toHaveBeenCalledWith('File limit reached: 1 files');
});
});
+
+describe('partitionUploads', () => {
+ let files: Map<string, ExtendedFile>;
+ let endpointFileConfig: EndpointFileConfig;
+
+ beforeEach(() => {
+ files = new Map();
+ endpointFileConfig = makeEndpointConfig();
+ });
+
+ it('keeps the files that fit and skips only the ones over the individual limit', () => {
+ const limit = 20 * megabyte;
+ endpointFileConfig = makeEndpointConfig({ fileSizeLimit: limit });
+ const fileList = [
+ makeSizedFile('small.pdf', 'application/pdf', 1 * megabyte),
+ makeSizedFile('huge.pdf', 'application/pdf', 21 * megabyte),
+ makeSizedFile('medium.pdf', 'application/pdf', 5 * megabyte),
+ ];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([0, 2]);
+ expect(skipped).toEqual([{ index: 1, file: fileList[1], reason: 'fileSize' }]);
+ });
+
+ it('skips every file when all of them are over the individual limit', () => {
+ endpointFileConfig = makeEndpointConfig({ fileSizeLimit: 1 * megabyte });
+ const fileList = [
+ makeSizedFile('one.pdf', 'application/pdf', 2 * megabyte),
+ makeSizedFile('two.pdf', 'application/pdf', 3 * megabyte),
+ ];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([]);
+ expect(skipped.map(({ reason }) => reason)).toEqual(['fileSize', 'fileSize']);
+ });
+
+ it('treats a file matching an existing attachment as a duplicate without dropping the rest', () => {
+ files = new Map([
+ [
+ 'f1',
+ makeExtendedFile({
+ file_id: 'f1',
+ filename: 'report.pdf',
+ size: 1024,
+ type: 'application/pdf',
+ }),
+ ],
+ ]);
+ const fileList = [
+ makeSizedFile('report.pdf', 'application/pdf', 1024),
+ makeSizedFile('notes.pdf', 'application/pdf', 2048),
+ ];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([1]);
+ expect(skipped).toEqual([{ index: 0, file: fileList[0], reason: 'duplicate' }]);
+ });
+
+ it('skips a file repeated within the same selection and keeps the first copy', () => {
+ const fileList = [
+ makeSizedFile('report.pdf', 'application/pdf', 1024),
+ makeSizedFile('report.pdf', 'application/pdf', 1024),
+ ];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([0]);
+ expect(skipped).toEqual([{ index: 1, file: fileList[1], reason: 'duplicate' }]);
+ });
+
+ it('leaves size checks alone when they are deferred until after transformation', () => {
+ endpointFileConfig = makeEndpointConfig({ fileSizeLimit: 1 * megabyte });
+ const fileList = [makeSizedFile('huge.pdf', 'application/pdf', 21 * megabyte)];
+
+ const { keptIndices, skipped } = partitionUploads({
+ files,
+ fileList,
+ endpointFileConfig,
+ skipSizeValidation: true,
+ });
+
+ expect(keptIndices).toEqual([0]);
+ expect(skipped).toEqual([]);
+ });
+
+ it('keeps everything when no individual limit is configured', () => {
+ endpointFileConfig = makeEndpointConfig({ fileSizeLimit: 0 });
+ const fileList = [makeSizedFile('huge.pdf', 'application/pdf', 500 * megabyte)];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([0]);
+ expect(skipped).toEqual([]);
+ });
+
+ it('leaves the batch-wide total limit to validateFileSizes', () => {
+ endpointFileConfig = makeEndpointConfig({
+ fileSizeLimit: 10 * megabyte,
+ totalSizeLimit: 7 * megabyte,
+ });
+ const fileList = [
+ makeSizedFile('one.pdf', 'application/pdf', 4 * megabyte),
+ makeSizedFile('two.pdf', 'application/pdf', 4 * megabyte),
+ ];
+
+ const { keptIndices, skipped } = partitionUploads({ files, fileList, endpointFileConfig });
+
+ expect(keptIndices).toEqual([0, 1]);
+ expect(skipped).toEqual([]);
+ });
+});
diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts
index 462b0145e5..99827472ab 100644
--- a/client/src/utils/files.ts
+++ b/client/src/utils/files.ts
@@ -250,18 +250,32 @@ type FileSizeValidationParams = {
endpointFileConfig: EndpointFileConfig;
};
+/** Identity used to detect a file already selected or attached: name, byte size, and MIME group. */
+const getFileSignature = (
+ name: string | undefined,
+ size: number | undefined,
+ type: string | undefined,
+): string => `${name}-${size}-${type?.split('/')[0] ?? 'file'}`;
+
+/** Normalizes the configured per-file cap: absent, zero, and negative all mean "no limit". */
+const getFileSizeLimit = ({ fileSizeLimit }: EndpointFileConfig): number | null =>
+ fileSizeLimit != null && fileSizeLimit > 0 ? fileSizeLimit : null;
+
export const validateFileSizes = ({
files,
fileList,
setError,
endpointFileConfig,
}: FileSizeValidationParams): boolean => {
- const { fileSizeLimit, totalSizeLimit } = endpointFileConfig;
-
- for (const file of fileList) {
- if (fileSizeLimit && file.size >= fileSizeLimit) {
- setError(`File size limit exceeded: ${fileSizeLimit / megabyte} MB`);
- return false;
+ const { totalSizeLimit } = endpointFileConfig;
+ const fileSizeLimit = getFileSizeLimit(endpointFileConfig);
+
+ if (fileSizeLimit != null) {
+ for (const file of fileList) {
+ if (file.size >= fileSizeLimit) {
+ setError(`File size limit exceeded: ${fileSizeLimit / megabyte} MB`);
+ return false;
+ }
}
}
@@ -280,6 +294,87 @@ export const validateFileSizes = ({
return true;
};
+export const validateFileLimit = ({
+ files,
+ fileList,
+ setError,
+ endpointFileConfig,
+}: FileSizeValidationParams): boolean => {
+ const { fileLimit } = endpointFileConfig;
+ if (fileLimit && fileList.length + files.size > fileLimit) {
+ setError(`File limit reached: ${fileLimit} files`);
+ return false;
+ }
+ return true;
+};
+
+export type UploadSkipReason = 'duplicate' | 'fileSize';
+
+export type SkippedUpload = {
+ /** Position in the `fileList` handed to `partitionUploads`, so callers can map back to their own parallel arrays */
+ index: number;
+ file: File;
+ reason: UploadSkipReason;
+};
+
+export type UploadPartition = {
+ keptIndices: number[];
+ skipped: SkippedUpload[];
+};
+
+/**
+ * Splits a selection into the files that may be uploaded and the ones that cannot, so a single
+ * offender no longer rejects everything picked alongside it. Duplicates are matched against files
+ * already attached and against earlier entries in the same selection. Only per-file rules belong
+ * here: `totalSizeLimit` is a property of the batch as a whole, so callers still run
+ * `validateFileSizes` over whatever survives.
+ */
+export const partitionUploads = ({
+ files,
+ fileList,
+ endpointFileConfig,
+ skipSizeValidation = false,
+}: {
+ fileList: File[];
+ files: Map<string, ExtendedFile>;
+ endpointFileConfig: EndpointFileConfig;
+ skipSizeValidation?: boolean;
+}): UploadPartition => {
+ const fileSizeLimit = skipSizeValidation ? null : getFileSizeLimit(endpointFileConfig);
+ const keptIndices: number[] = [];
+ const skipped: SkippedUpload[] = [];
+
+ const signatures = new Set<string>();
+ for (const existingFile of files.values()) {
+ signatures.add(
+ getFileSignature(
+ existingFile.file?.name ?? existingFile.filename,
+ existingFile.size,
+ existingFile.type,
+ ),
+ );
+ }
+
+ for (let i = 0; i < fileList.length; i++) {
+ const file = fileList[i];
+ const signature = getFileSignature(file.name, file.size, file.type);
+ if (signatures.has(signature)) {
+ skipped.push({ index: i, file, reason: 'duplicate' });
+ continue;
+ }
+ signatures.add(signature);
+
+ if (fileSizeLimit != null && file.size >= fileSizeLimit) {
+ skipped.push({ index: i, file, reason: 'fileSize' });
+ continue;
+ }
+
+ keptIndices.push(i);
+ }
+
+ return { keptIndices, skipped };
+};
+
type FileDuplicateValidationParams = {
fileList: File[];
files: Map<string, ExtendedFile>;
@@ -292,13 +387,11 @@ export const validateFileDuplicates = ({
setError,
}: FileDuplicateValidationParams): boolean => {
const combinedFilesInfo = [
- ...Array.from(files.values()).map(
- (file) =>
- `${file.file?.name ?? file.filename}-${file.size}-${file.type?.split('/')[0] ?? 'file'}`,
+ ...Array.from(files.values()).map((file) =>
+ getFileSignature(file.file?.name ?? file.filename, file.size, file.type),
),
- ...fileList.map(
- (file: File | undefined) =>
- `${file?.name}-${file?.size}-${file?.type.split('/')[0] ?? 'file'}`,
+ ...fileList.map((file: File | undefined) =>
+ getFileSignature(file?.name, file?.size, file?.type),
),
];
@@ -320,6 +413,7 @@ export const validateFiles = ({
toolResource,
fileConfig,
skipSizeValidation = false,
+ skipBatchRules = false,
}: {
fileList: File[];
files: Map<string, ExtendedFile>;
@@ -327,9 +421,13 @@ export const validateFiles = ({
endpointFileConfig: EndpointFileConfig;
toolResource?: string;
fileConfig: FileConfig | null;
+ /** Defer size checks to `partitionUploads` once processing has settled each file's final bytes */
skipSizeValidation?: boolean;
+ /** The caller partitions the selection itself, so the rules that would reject the batch as a
+ * whole — the file count and duplicates — wait until it has dropped what it can */
+ skipBatchRules?: boolean;
}) => {
- const { fileLimit, supportedMimeTypes, disabled } = endpointFileConfig;
+ const { supportedMimeTypes, disabled } = endpointFileConfig;
/** Block all uploads if the endpoint is explicitly disabled */
if (disabled === true) {
setError('com_ui_attach_error_disabled');
@@ -341,8 +439,7 @@ export const validateFiles = ({
return false;
}
- if (fileLimit && fileList.length + files.size > fileLimit) {
- setError(`File limit reached: ${fileLimit} files`);
+ if (!skipBatchRules && !validateFileLimit({ files, fileList, setError, endpointFileConfig })) {
return false;
}
@@ -385,6 +482,10 @@ export const validateFiles = ({
return false;
}
+ if (skipBatchRules) {
+ return true;
+ }
+
return validateFileDuplicates({ files, fileList, setError });
};