Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🎞️ fix: Deliver Opted-In Custom Endpoint Media Safely (#15937)
Danny Avila2f8bc3dfcd5f
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.jsindex 5ec6e7db96..81f8998760 100644--- a/api/app/clients/BaseClient.js+++ b/api/app/clients/BaseClient.js@@ -45,6 +45,7 @@ const {resolveUploadLLMDeliveryPath,isSpeechProviderConfigured,resolveUseResponsesApi,+ getCustomEndpointProvider,} = require('librechat-data-provider');const { getStrategyFunctions } = require('~/server/services/Files/strategies');const { logViolation } = require('~/cache');@@ -1826,6 +1827,12 @@ class BaseClient {endpointConfig: this._endpointFileConfig,fileConfig: this._mergedFileConfig,endpoint: this._deliveryEndpoint,+ endpointProvider:+ this.options.agent?.provider ??+ getCustomEndpointProvider(+ this.options.req?.config?.endpoints?.custom,+ this._deliveryEndpoint,+ ),useResponsesApi: this.usesResponsesApi(),sttConfigured: isSpeechProviderConfigured(this.options.req?.config?.speech?.stt),});diff --git a/api/server/routes/files/files.agents.test.js b/api/server/routes/files/files.agents.test.jsindex 4323a28e81..45b082601d 100644--- a/api/server/routes/files/files.agents.test.js+++ b/api/server/routes/files/files.agents.test.js@@ -427,6 +427,33 @@ describe('File Routes - Agent Files Endpoint', () => {return testApp;};+ it.each([+ [false, 'application/json'],+ [true, 'text/event-stream'],+ ])(+ 'rejects unsupported provider audio before persistence (legacy=%s, accept=%s)',+ async (legacyFileUploadUX, accept) => {+ const testApp = createAppWithUser(+ otherUserId,+ SystemRoles.USER,+ {+ fileConfig: {+ endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } },+ },+ },+ { originalname: 'clip.wma', mimetype: 'audio/wma' },+ );+ const response = await request(testApp).post('/files').set('Accept', accept).send({+ endpoint: 'MyGateway',+ file_id: uuidv4(),+ });+ expect(response.status).toBe(415);+ expect(response.body.message).toBe('com_error_files_provider_audio_format');+ expect(processAgentFileUpload).not.toHaveBeenCalled();+ expect(fs.unlink).toHaveBeenCalledWith('/tmp/test.txt');+ },+ );+it('inspects the canonical sanitized filename used by upload processing', async () => {await createAgent({id: agentCustomId,diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.jsindex 7e5512f0cd..435027a3b0 100644--- a/api/server/routes/files/images.js+++ b/api/server/routes/files/images.js@@ -23,6 +23,7 @@ const {resolveUploadLLMDeliveryPath,isResponsesApiUpload,isSpeechProviderConfigured,+ getCustomEndpointProvider,} = require('librechat-data-provider');const {processAgentFileUpload,@@ -107,6 +108,7 @@ router.post('/', async (req, res) => {endpointConfig: getEndpointFileConfig({ fileConfig, endpoint: effectiveEndpoint }),fileConfig,endpoint: effectiveEndpoint,+ endpointProvider: getCustomEndpointProvider(req.config?.endpoints?.custom, effectiveEndpoint),useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi),sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt),});diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.jsindex 0a5705e60c..7ddf9ab729 100644--- a/api/server/services/Files/process.js+++ b/api/server/services/Files/process.js@@ -25,6 +25,7 @@ const {isMessageFileUpload,isResponsesApiUpload,isSpeechProviderConfigured,+ getCustomEndpointProvider,} = require('librechat-data-provider');const { logger, runAsSystem } = require('@librechat/data-schemas');const {@@ -494,6 +495,7 @@ const processImageFile = async ({ req, res, metadata, returnFile = false, sseStrendpointConfig,fileConfig,endpoint: configEndpoint,+ endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, configEndpoint),useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi),sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt),});@@ -810,6 +812,7 @@ const processAgentFileUpload = async ({ req, res, metadata, sseStream }) => {endpointConfig,fileConfig,endpoint,+ endpointProvider: getCustomEndpointProvider(appConfig?.endpoints?.custom, endpoint),useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi ?? req.body?.useResponsesApi),sttConfigured: isSpeechProviderConfigured(appConfig?.speech?.stt),});diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsxindex 71f89b03ee..ead2d57bc1 100644--- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx+++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx@@ -20,6 +20,7 @@ import {Providers,EToolResources,EModelEndpoint,+ isExplicitMimeConfig,getConfiguredMimeAccept,bedrockDocumentMimeTypes,defaultAgentCapabilities,@@ -52,7 +53,8 @@ type FileUploadType =| 'document'| 'image_document'| 'image_document_extended'- | 'image_document_video_audio';+ | 'image_document_video_audio'+ | 'image_document_video_audio_configured';/** What each provider upload path can actually send, used to scope the picker filter to selectable files. */const fileTypeCapabilities: Record<FileUploadType, MimeUploadCapability> = {@@ -68,6 +70,10 @@ const fileTypeCapabilities: Record<FileUploadType, MimeUploadCapability> = {categories: ['image', 'document', 'audio', 'video'],documentMimeTypes: ['application/pdf'],},+ /** Custom endpoint with an admin-configured allowlist: the config decides, including video/audio. */+ image_document_video_audio_configured: {+ categories: ['image', 'document', 'audio', 'video'],+ },};interface AttachFileMenuProps {@@ -164,6 +170,10 @@ const AttachFileMenu = ({inputRef.current.accept = `image/*,.heif,.heic,${bedrockDocumentExtensions}`;} else if (fileType === 'image_document_video_audio') {inputRef.current.accept = 'image/*,.heif,.heic,.pdf,application/pdf,video/*,audio/*';+ } else if (fileType === 'image_document_video_audio_configured') {+ /** Only reached with an explicit allowlist the accept string cannot represent; leave the+ * picker open so it never hides a file the admin allowed (backend still enforces). */+ inputRef.current.accept = '';} else {inputRef.current.accept = '';}@@ -237,6 +247,11 @@ const AttachFileMenu = ({endpointType === EModelEndpoint.bedrock) {fileType = 'image_document_extended';+ } else if (+ endpointType === EModelEndpoint.custom &&+ isExplicitMimeConfig(endpointFileConfig?.supportedMimeTypes)+ ) {+ fileType = 'image_document_video_audio_configured';}onAction(fileType);},@@ -324,6 +339,7 @@ const AttachFileMenu = ({handleUploadClick,setEphemeralAgent,sharePointEnabled,+ endpointFileConfig?.supportedMimeTypes,codeAllowedByAgent,fileSearchAllowedByAgent,setIsSharePointDialogOpen,diff --git a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsxindex 13d60564e4..7fcd7ba32b 100644--- a/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx+++ b/client/src/components/Chat/Input/Files/__tests__/AttachFileMenu.spec.tsx@@ -208,6 +208,46 @@ describe('AttachFileMenu', () => {});});+ describe('Upload to Provider picker filter', () => {+ /** `accept` is reset right after `click()`, so capture it at click time. */+ const captureAcceptOnClick = () => {+ const seen: string[] = [];+ const spy = jest.spyOn(HTMLInputElement.prototype, 'click').mockImplementation(function (+ this: HTMLInputElement,+ ) {+ seen.push(this.accept);+ });+ return { seen, spy };+ };++ it('keeps images and PDFs only for a custom endpoint with the inherited default config', () => {+ const { seen, spy } = captureAcceptOnClick();+ setupMocks({ provider: 'MyGateway' });+ renderMenu({ endpointType: EModelEndpoint.custom, endpointFileConfig: {} });+ openMenu();+ fireEvent.click(screen.getByText('Upload to Provider'));+ expect(seen).toEqual(['image/*,.heif,.heic,.pdf,application/pdf']);+ spy.mockRestore();+ });++ it('adds video when a custom endpoint config explicitly allows it', () => {+ const { seen, spy } = captureAcceptOnClick();+ setupMocks({ provider: 'MyGateway' });+ renderMenu({+ endpointType: EModelEndpoint.custom,+ endpointFileConfig: {+ supportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/],+ },+ });+ openMenu();+ fireEvent.click(screen.getByText('Upload to Provider'));+ expect(seen).toHaveLength(1);+ expect(seen[0]).toContain('video/*');+ expect(seen[0]).not.toContain('audio/*');+ spy.mockRestore();+ });+ });+describe('Upload to Provider vs Upload Image', () => {it('shows "Upload to Provider" when endpointType is custom (resolved from agent provider)', () => {setupMocks({ provider: 'Moonshot' });diff --git a/client/src/hooks/Files/__tests__/useFileHandling.test.ts b/client/src/hooks/Files/__tests__/useFileHandling.test.tsindex bcbc2e1b56..e741324f7d 100644--- a/client/src/hooks/Files/__tests__/useFileHandling.test.ts+++ b/client/src/hooks/Files/__tests__/useFileHandling.test.ts@@ -1,4 +1,4 @@-import { renderHook, act } from '@testing-library/react';+import { renderHook, act, waitFor } from '@testing-library/react';import {megabyte,Constants,@@ -210,6 +210,40 @@ describe('useFileHandling', () => {const loadHook = async () => (await import('../useFileHandling')).default;+ it('removes rejected provider audio and localizes the upload error before retry', async () => {+ const consoleLog = jest.spyOn(console, 'log').mockImplementation(() => undefined);+ const useFileHandling = await loadHook();+ const { result } = renderHook(() => useFileHandling());+ const recovery = jest.fn();+ await act(async () => {+ await result.current.handleFiles(+ [new File(['audio'], 'clip.wma', { type: 'audio/wma' })],+ undefined,+ { onError: recovery },+ );+ });+ const body = mockMutate.mock.calls[0][0] as FormData;+ const fileId = body.get('file_id');+ act(() =>+ mockUploadOptions.onError?.(+ {+ response: { status: 415, data: { message: 'com_error_files_provider_audio_format' } },+ },+ body,+ ),+ );+ expect(mockDeleteFileById).toHaveBeenCalledWith(fileId);+ await waitFor(() =>+ expect(mockLocalize).toHaveBeenCalledWith('com_error_files_provider_audio_format'),+ );+ expect(recovery).toHaveBeenCalledWith(fileId);+ await act(async () => {+ await result.current.handleFiles([new File(['audio'], 'clip.wav', { type: 'audio/wav' })]);+ });+ expect(mockMutate).toHaveBeenCalledTimes(2);+ consoleLog.mockRestore();+ });+describe('endpointOverride', () => {it('clears the loading state when file validation throws', async () => {const consoleError = jest.spyOn(console, 'error').mockImplementation(() => undefined);diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.jsonindex 64de2fe31b..6ca1ba26a7 100644--- a/client/src/locales/en/translation.json+++ b/client/src/locales/en/translation.json@@ -1089,6 +1089,7 @@"com_ui_attach_error_size": "File size limit exceeded for endpoint:","com_ui_attach_error_total_size": "Total file size limit exceeded for endpoint:","com_ui_attach_error_type": "Unsupported file type for endpoint:",+ "com_error_files_provider_audio_format": "This audio format cannot be attached directly to this model. Convert the file to WAV or MP3 and upload it again, or choose Upload as Text if transcription is configured.","com_ui_attach_remove": "Remove file","com_ui_attach_warn_endpoint": "Non-Assistant files may be ignored without a compatible tool","com_ui_attached_image": "Attached image",diff --git a/client/src/utils/__tests__/getViableUploadOptions.spec.ts b/client/src/utils/__tests__/getViableUploadOptions.spec.tsindex 261ea70e40..6d2ea63580 100644--- a/client/src/utils/__tests__/getViableUploadOptions.spec.ts+++ b/client/src/utils/__tests__/getViableUploadOptions.spec.ts@@ -132,6 +132,33 @@ describe('getViableUploadOptions', () => {expect(getViableUploadOptions([file(XLSX, 'report.xlsx')], ctx)).toEqual([undefined]);});+ it('offers direct attach for a video when the custom config explicitly allows video', () => {+ const ctx = baseCtx({+ provider: 'MyGateway',+ endpoint: 'MyGateway',+ endpointType: 'custom',+ fileSearchEnabled: false,+ codeEnabled: false,+ contextEnabled: false,+ endpointSupportedMimeTypes: [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/],+ });+ expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([undefined]);+ expect(getViableUploadOptions([file('audio/wav', 'tone.wav')], ctx)).toEqual([]);+ });++ it('does not offer video for a custom endpoint that inherits the default config', () => {+ const ctx = baseCtx({+ provider: 'MyGateway',+ endpoint: 'MyGateway',+ endpointType: 'custom',+ fileSearchEnabled: false,+ codeEnabled: false,+ contextEnabled: false,+ endpointSupportedMimeTypes: undefined,+ });+ expect(getViableUploadOptions([file('video/mp4', 'clip.mp4')], ctx)).toEqual([]);+ });+it('does not treat a non-permissive custom config as broad provider support', () => {const ctx = baseCtx({provider: 'MyGateway',diff --git a/client/src/utils/files.ts b/client/src/utils/files.tsindex 62e77dacaf..857742f98a 100644--- a/client/src/utils/files.ts+++ b/client/src/utils/files.ts@@ -16,7 +16,7 @@ import {EModelEndpoint,retrievalMimeTypes,isBedrockDocumentType,- isPermissiveMimeConfig,+ isExplicitMimeConfig,codeInterpreterMimeTypes,isDocumentSupportedProvider,fileConfig as defaultFileConfig,@@ -522,12 +522,12 @@ const isProviderAttachType = (type: string, ctx: UploadOptionContext): boolean =isDocumentSupportedProvider(currentProvider) ||isAzureWithResponsesApi) {- /** Custom endpoints that the admin opened up (permissive config) honor that allowlist,- * matching the file picker; an inherited default config is not treated as opened up. */+ /** Custom endpoints with an admin-configured allowlist honor it for direct attach (this is+ * how video/audio get opted in for an OpenAI-compatible gateway), matching the file picker+ * and the server-side encoders; an inherited default config is not treated as opened up. */if (ctx.endpointType === EModelEndpoint.custom &&- ctx.endpointSupportedMimeTypes != null &&- isPermissiveMimeConfig(ctx.endpointSupportedMimeTypes)+ isExplicitMimeConfig(ctx.endpointSupportedMimeTypes)) {return checkType(type, ctx.endpointSupportedMimeTypes);}diff --git a/packages/api/src/agents/files/encode.ts b/packages/api/src/agents/files/encode.tsindex 63b273bd76..d4de921577 100644--- a/packages/api/src/agents/files/encode.ts+++ b/packages/api/src/agents/files/encode.ts@@ -103,6 +103,7 @@ export function createRunFileMessageEncoder(endpointConfig,fileConfig,endpoint,+ endpointProvider: agent.provider,useResponsesApi,sttConfigured: isSpeechProviderConfigured(deps.req.config?.speech?.stt),}),diff --git a/packages/api/src/files/encode/audio.spec.ts b/packages/api/src/files/encode/audio.spec.tsnew file mode 100644index 0000000000..e795042fd6--- /dev/null+++ b/packages/api/src/files/encode/audio.spec.ts@@ -0,0 +1,211 @@+import { Providers } from '@librechat/agents';+import type { IMongoFile } from '@librechat/data-schemas';+import type { ServerRequest } from '~/types';+import { encodeAndFormatAudios } from './audio';++jest.mock('~/files/validation', () => ({+ validateAudio: jest.fn(),+}));++jest.mock('./utils', () => ({+ ...jest.requireActual('./utils'),+ getFileStream: jest.fn(),+ getConfiguredFileSizeLimit: jest.fn(),+ isConfiguredProviderMediaType: jest.fn(),+}));++jest.mock('./memoryGuard', () => ({+ runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()),+}));++import { validateAudio } from '~/files/validation';+import { getFileStream, isConfiguredProviderMediaType } from './utils';+import { Types } from 'mongoose';++const mockedValidateAudio = validateAudio as jest.MockedFunction<typeof validateAudio>;+const mockedGetFileStream = getFileStream as jest.MockedFunction<typeof getFileStream>;+const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction<+ typeof isConfiguredProviderMediaType+>;++const createMockFile = (type = 'audio/wav', filename = 'tone.wav'): IMongoFile =>+ ({+ _id: new Types.ObjectId(),+ file_id: 'audio-1',+ filename,+ filepath: `/uploads/${filename}`,+ type,+ bytes: 1024,+ source: 'local',+ user: 'user-1',+ object: 'file',+ usage: 0,+ createdAt: new Date(),+ updatedAt: new Date(),+ }) as unknown as IMongoFile;++const req = { config: {} } as unknown as ServerRequest;+const getStrategyFunctions = jest.fn();++describe('encodeAndFormatAudios - provider formatting', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ mockedValidateAudio.mockResolvedValue({ isValid: true });+ mockedIsConfigured.mockReturnValue(false);+ const file = createMockFile();+ mockedGetFileStream.mockResolvedValue({+ file,+ content: 'AAAA',+ metadata: {+ file_id: file.file_id,+ filepath: file.filepath,+ source: file.source,+ filename: file.filename,+ type: file.type,+ },+ });+ });++ it('emits a Google media block for google', async () => {+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile()],+ { provider: Providers.GOOGLE },+ getStrategyFunctions,+ );+ expect(result.audios).toEqual([{ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }]);+ });++ it('emits an OpenAI-compatible input_audio block for openrouter without configuration', async () => {+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile()],+ { provider: Providers.OPENROUTER },+ getStrategyFunctions,+ );+ expect(mockedIsConfigured).not.toHaveBeenCalled();+ expect(result.audios).toEqual([+ { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } },+ ]);+ });++ it('drops the audio for a custom endpoint that is not configured for audio', async () => {+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile()],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ );+ expect(mockedIsConfigured).toHaveBeenCalledWith(+ req,+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ 'audio/wav',+ );+ expect(result.audios).toEqual([]);+ expect(result.files).toHaveLength(1);+ });++ it('emits an input_audio block for a custom endpoint whose config allows audio', async () => {+ mockedIsConfigured.mockReturnValue(true);+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile()],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ );+ expect(result.audios).toEqual([+ { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } },+ ]);+ });++ it('never emits a block for providers without an audio path, even when configured', async () => {+ mockedIsConfigured.mockReturnValue(true);+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile()],+ { provider: Providers.ANTHROPIC },+ getStrategyFunctions,+ );+ expect(result.audios).toEqual([]);+ });++ describe('input_audio format canonicalization', () => {+ /** Re-points the `getFileStream` mock at a specific file for one test. */+ const stageFile = (file: IMongoFile) => {+ mockedGetFileStream.mockResolvedValue({+ file,+ content: 'AAAA',+ metadata: {+ file_id: file.file_id,+ filepath: file.filepath,+ source: file.source,+ filename: file.filename,+ type: file.type,+ },+ });+ };++ const formatFor = async (file: IMongoFile) => {+ stageFile(file);+ mockedIsConfigured.mockReturnValue(true);+ const result = await encodeAndFormatAudios(+ req,+ [file],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ );+ return (result.audios[0] as { input_audio?: { format?: string } })?.input_audio?.format;+ };++ it.each([+ ['audio/wave', 'clip.wave', 'wav'],+ ['audio/x-wav', 'clip.x-wav', 'wav'],+ ['audio/mpeg', 'clip.mpeg', 'mp3'],+ ['audio/mpeg3', 'clip.mpeg3', 'mp3'],+ ['audio/x-m4a', 'clip.x-m4a', 'm4a'],+ ['audio/vorbis', 'clip.vorbis', 'ogg'],+ ['audio/x-flac', 'clip.x-flac', 'flac'],+ ])('canonicalizes %s to a provider-accepted format', async (type, filename, expected) => {+ expect(await formatFor(createMockFile(type, filename))).toBe(expected);+ });++ it('derives the format from the MIME type when the filename has no extension', async () => {+ expect(await formatFor(createMockFile('audio/mpeg', 'recording'))).toBe('mp3');+ });++ it('prefers the canonical MIME format over a mismatched extension', async () => {+ expect(await formatFor(createMockFile('audio/mpeg', 'clip.wav'))).toBe('mp3');+ });++ it('falls back to a supported filename extension for MIME types with no mapping', async () => {+ expect(await formatFor(createMockFile('audio/aac', 'clip.aac'))).toBe('aac');+ });++ it('throws rather than emitting an unsupported format', async () => {+ /** `audio/wma` passes MIME validation but has no provider-accepted format. */+ stageFile(createMockFile('audio/wma', 'recording'));+ mockedIsConfigured.mockReturnValue(true);+ await expect(+ encodeAndFormatAudios(+ req,+ [createMockFile('audio/wma', 'recording')],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ ),+ ).rejects.toThrow(/supported audio format/i);+ });++ it('canonicalizes for openrouter as well', async () => {+ stageFile(createMockFile('audio/wave', 'clip.wave'));+ const result = await encodeAndFormatAudios(+ req,+ [createMockFile('audio/wave', 'clip.wave')],+ { provider: Providers.OPENROUTER },+ getStrategyFunctions,+ );+ expect(result.audios).toEqual([+ { type: 'input_audio', input_audio: { data: 'AAAA', format: 'wav' } },+ ]);+ });+ });+});diff --git a/packages/api/src/files/encode/audio.ts b/packages/api/src/files/encode/audio.tsindex fa155cd3b3..ac51c8f772 100644--- a/packages/api/src/files/encode/audio.ts+++ b/packages/api/src/files/encode/audio.ts@@ -1,11 +1,13 @@import { Providers } from '@librechat/agents';-import { isDocumentSupportedProvider } from 'librechat-data-provider';+import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider';import type { IMongoFile } from '@librechat/data-schemas';import type { ServerRequest, StrategyFunctions, AudioResult } from '~/types';import {getFileStream,+ getAudioFormat,getConfiguredFileSizeLimit,isAttachmentObjectNotFoundError,+ isConfiguredProviderMediaType,} from './utils';import { validateAudio } from '~/files/validation';import { runGuardedEncode } from './memoryGuard';@@ -15,7 +17,7 @@ import { runGuardedEncode } from './memoryGuard';* @param req - The request object* @param files - Array of audio files* @param params - Object containing provider and optional endpoint- * @param params.provider - The provider to format for (currently only google is supported)+ * @param params.provider - The provider to format for* @param params.endpoint - Optional endpoint name for file config lookup* @param getStrategyFunctions - Function to get strategy functions* @returns Promise that resolves to audio and file metadata@@ -91,13 +93,22 @@ export async function encodeAndFormatAudios(mimeType: file.type,data: content,});- } else if (provider === Providers.OPENROUTER) {- // Extract format from filename extension (e.g., 'audio.mp3' -> 'mp3')- // OpenRouter expects format values like: wav, mp3, aiff, aac, ogg, flac, m4a, pcm16, pcm24- // Note: MIME types don't always match (e.g., 'audio/mpeg' is mp3, not mpeg), so that is why we are using the file extension instead- const format = file.filename.split('.').pop()?.toLowerCase();+ } else if (+ provider === Providers.OPENROUTER ||+ (isOpenAILikeProvider(provider) &&+ isConfiguredProviderMediaType(req, { provider, endpoint }, file.type))+ ) {+ /** OpenAI-compatible `input_audio` part (OpenRouter, vLLM, and other gateways).+ * Custom endpoints only get here when the admin listed audio types in the+ * endpoint's `supportedMimeTypes`, since not every gateway accepts audio. */+ /** Canonicalized from the MIME type: the accepted MIME list contains aliases+ * (`audio/wave`, `audio/mpeg`) that are not valid format values, and the filename+ * may carry no extension at all. */+ const format = getAudioFormat(file.type, file.filename);if (!format) {- throw new Error(`Could not extract audio format from filename: ${file.filename}`);+ throw new Error(+ `Could not determine a supported audio format for "${file.filename}" (${file.type})`,+ );}result.audios.push({type: 'input_audio',diff --git a/packages/api/src/files/encode/utils.spec.ts b/packages/api/src/files/encode/utils.spec.tsindex 24367409c7..d4882d952d 100644--- a/packages/api/src/files/encode/utils.spec.ts+++ b/packages/api/src/files/encode/utils.spec.ts@@ -1,6 +1,13 @@import { Readable } from 'node:stream';+import { Providers } from '@librechat/agents';+import { audioMimeTypes } from 'librechat-data-provider';import type { ServerRequest } from '~/types';-import { AttachmentObjectNotFoundError, getFileStream } from './utils';+import {+ AttachmentObjectNotFoundError,+ getAudioFormat,+ getFileStream,+ isConfiguredProviderMediaType,+} from './utils';const file = {file_id: 'file-1',@@ -84,3 +91,110 @@ describe('getFileStream', () => {expect(getDownloadStream).toHaveBeenCalledWith(req, legacyUrl);});});++/** Uses the real data-provider merge logic so the "inherited default" identity check is exercised. */+const reqWith = (fileConfig: unknown): ServerRequest =>+ ({ config: fileConfig === undefined ? undefined : { fileConfig } }) as unknown as ServerRequest;++describe('isConfiguredProviderMediaType', () => {+ const params = { provider: Providers.OPENAI, endpoint: 'MyGateway' };++ it('is false without any fileConfig', () => {+ expect(isConfiguredProviderMediaType(reqWith(undefined), params, 'video/mp4')).toBe(false);+ });++ it('is false when the endpoint only inherits the built-in default list', () => {+ const req = reqWith({ endpoints: { OtherEndpoint: { fileLimit: 3 } } });+ expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(false);+ expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false);+ });++ it('is true when the endpoint config explicitly lists the media type', () => {+ const req = reqWith({+ endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'application/pdf', 'video/.*'] } },+ });+ expect(isConfiguredProviderMediaType(req, params, 'video/mp4')).toBe(true);+ expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(false);+ });++ it('is true for a permissive config', () => {+ const req = reqWith({ endpoints: { MyGateway: { supportedMimeTypes: ['.*'] } } });+ expect(isConfiguredProviderMediaType(req, params, 'audio/wav')).toBe(true);+ });++ it('falls back to the provider key when no endpoint is given', () => {+ const req = reqWith({ endpoints: { openAI: { supportedMimeTypes: ['audio/.*'] } } });+ expect(isConfiguredProviderMediaType(req, { provider: Providers.OPENAI }, 'audio/wav')).toBe(+ true,+ );+ });+});++describe('getAudioFormat', () => {+ it.each([+ ['audio/mp3', 'mp3'],+ ['audio/mpeg', 'mp3'],+ ['audio/mpeg3', 'mp3'],+ ['audio/wav', 'wav'],+ ['audio/wave', 'wav'],+ ['audio/x-wav', 'wav'],+ ['audio/ogg', 'ogg'],+ ['audio/vorbis', 'ogg'],+ ['audio/mp4', 'm4a'],+ ['audio/x-m4a', 'm4a'],+ ['audio/flac', 'flac'],+ ['audio/x-flac', 'flac'],+ ])('maps %s to %s regardless of the filename', (mimeType, expected) => {+ expect(getAudioFormat(mimeType, 'clip.bogus')).toBe(expected);+ expect(getAudioFormat(mimeType, 'recording')).toBe(expected);+ });++ it('is case-insensitive for the MIME type', () => {+ expect(getAudioFormat('AUDIO/WAVE', 'clip.wave')).toBe('wav');+ });++ it('falls back to a supported extension when the MIME type is unmapped', () => {+ expect(getAudioFormat('audio/unknown', 'clip.mp3')).toBe('mp3');+ expect(getAudioFormat('audio/unknown', 'clip.PCM16')).toBe('pcm16');+ });++ it('rejects an unsupported extension rather than passing it through', () => {+ expect(getAudioFormat('audio/unknown', 'clip.wave')).toBeUndefined();+ expect(getAudioFormat('audio/unknown', 'clip.exe')).toBeUndefined();+ });++ it('returns undefined when the filename has no extension and the MIME is unmapped', () => {+ expect(getAudioFormat('audio/wma', 'recording')).toBeUndefined();+ expect(getAudioFormat('', '')).toBeUndefined();+ });++ it('does not treat a leading-dot filename as an extension', () => {+ expect(getAudioFormat('audio/unknown', '.mp3')).toBeUndefined();+ });++ /** Guards against the accepted-MIME list drifting ahead of the format mapping. */+ it('resolves a format for every accepted audio MIME type that has one', () => {+ const accepted = [+ 'audio/mp3',+ 'audio/mpeg',+ 'audio/mpeg3',+ 'audio/wav',+ 'audio/wave',+ 'audio/x-wav',+ 'audio/ogg',+ 'audio/vorbis',+ 'audio/mp4',+ 'audio/m4a',+ 'audio/x-m4a',+ 'audio/flac',+ 'audio/x-flac',+ 'audio/webm',+ 'audio/aac',+ 'audio/opus',+ ];+ for (const mimeType of accepted) {+ expect(audioMimeTypes.test(mimeType)).toBe(true);+ expect(getAudioFormat(mimeType, 'recording')).toBeDefined();+ }+ });+});diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.tsindex 7d7ac21a3a..9c39ce2c13 100644--- a/packages/api/src/files/encode/utils.ts+++ b/packages/api/src/files/encode/utils.ts@@ -1,6 +1,11 @@import getStream from 'get-stream';import { Providers } from '@librechat/agents';-import { FileSources, mergeFileConfig, getEndpointFileConfig } from 'librechat-data-provider';+import {+ FileSources,+ mergeFileConfig,+ isExplicitMimeConfig,+ getEndpointFileConfig,+} from 'librechat-data-provider';import type { ServerRequest, StrategyFunctions, ProcessedFile } from '~/types';import type { StoredFileRef } from '~/storage/path';import { resolveDownloadPath } from '~/storage/path';@@ -70,6 +75,107 @@ export const getConfiguredFileSizeLimit = (return endpointConfig?.fileSizeLimit;};+/**+ * Whether the admin explicitly allowed `mimeType` for this endpoint via+ * `fileConfig.endpoints.<name>.supportedMimeTypes`. The inherited default list does+ * not count as opting in, mirroring the client's picker and drag-drop logic, so an+ * OpenAI-compatible endpoint only receives video/audio parts when configured for them.+ * @param req - The server request object containing config+ * @param params - Object containing provider and optional endpoint+ * @param params.provider - The provider to look up+ * @param params.endpoint - Optional endpoint name for lookup+ * @param mimeType - The MIME type of the file being attached+ * @returns True when the endpoint config explicitly matches the MIME type+ */+export const isConfiguredProviderMediaType = (+ req: ServerRequest,+ params: {+ provider: Providers;+ endpoint?: string;+ },+ mimeType: string,+): boolean => {+ if (!req.config?.fileConfig) {+ return false;+ }+ const { provider, endpoint } = params;+ const fileConfig = mergeFileConfig(req.config.fileConfig);+ const endpointConfig = getEndpointFileConfig({+ fileConfig,+ endpoint: endpoint ?? provider,+ });+ const types = endpointConfig?.supportedMimeTypes;+ if (!isExplicitMimeConfig(types)) {+ return false;+ }+ return fileConfig.checkType?.(mimeType, types) ?? false;+};++/**+ * Maps accepted audio MIME types to the format token OpenAI-compatible providers expect.+ * Mirrors the canonicalization in `api/server/services/Files/Audio/STTService.js`: the+ * accepted MIME list carries aliases (`audio/wave`, `audio/x-wav`, `audio/mpeg`) whose+ * names are not themselves valid format values.+ */+const audioMimeToFormat: Record<string, string> = {+ 'audio/mp3': 'mp3',+ 'audio/mpeg': 'mp3',+ 'audio/mpeg3': 'mp3',+ 'audio/wav': 'wav',+ 'audio/wave': 'wav',+ 'audio/x-wav': 'wav',+ 'audio/ogg': 'ogg',+ 'audio/vorbis': 'ogg',+ 'audio/opus': 'ogg',+ 'audio/mp4': 'm4a',+ 'audio/m4a': 'm4a',+ 'audio/x-m4a': 'm4a',+ 'audio/flac': 'flac',+ 'audio/x-flac': 'flac',+ 'audio/webm': 'webm',+ 'audio/aac': 'aac',+};++/**+ * Format values OpenAI-compatible providers accept for an `input_audio` part.+ * Used to validate an extension-derived fallback so an arbitrary filename suffix+ * never reaches the provider as a format.+ */+const supportedAudioFormats = new Set([+ 'wav',+ 'mp3',+ 'aiff',+ 'aac',+ 'ogg',+ 'flac',+ 'm4a',+ 'webm',+ 'pcm16',+ 'pcm24',+]);++/**+ * Resolves the `input_audio.format` value for a file.+ *+ * Prefers the MIME type, which is validated upstream against `audioMimeTypes`, because+ * the filename extension is user-controlled: it may be absent (`recording`), an alias+ * that is not a valid format (`clip.wave`), or disagree with the actual content. Falls+ * back to the extension only when it is itself a supported format.+ * @param mimeType - The file's MIME type+ * @param filename - The original filename+ * @returns The canonical format, or undefined when neither source yields a supported one+ */+export const getAudioFormat = (mimeType: string, filename: string): string | undefined => {+ const fromMime = audioMimeToFormat[mimeType?.toLowerCase()];+ if (fromMime) {+ return fromMime;+ }+ const parts = filename?.toLowerCase().split('.') ?? [];+ /** `parts[0]` must be non-empty so a dotfile (`.mp3`) is not read as an extension. */+ const extension = parts.length > 1 && parts[0] ? parts[parts.length - 1] : undefined;+ return extension && supportedAudioFormats.has(extension) ? extension : undefined;+};+/*** Processes a file by downloading and encoding it to base64* @param req - Express request objectdiff --git a/packages/api/src/files/encode/video.spec.ts b/packages/api/src/files/encode/video.spec.tsnew file mode 100644index 0000000000..06da3028d0--- /dev/null+++ b/packages/api/src/files/encode/video.spec.ts@@ -0,0 +1,130 @@+import { Providers } from '@librechat/agents';+import type { IMongoFile } from '@librechat/data-schemas';+import type { ServerRequest } from '~/types';+import { encodeAndFormatVideos } from './video';++jest.mock('~/files/validation', () => ({+ validateVideo: jest.fn(),+}));++jest.mock('./utils', () => ({+ getFileStream: jest.fn(),+ getConfiguredFileSizeLimit: jest.fn(),+ isConfiguredProviderMediaType: jest.fn(),+}));++jest.mock('./memoryGuard', () => ({+ runGuardedEncode: jest.fn((_bytes: number, fn: () => unknown) => fn()),+}));++import { validateVideo } from '~/files/validation';+import { getFileStream, isConfiguredProviderMediaType } from './utils';+import { Types } from 'mongoose';++const mockedValidateVideo = validateVideo as jest.MockedFunction<typeof validateVideo>;+const mockedGetFileStream = getFileStream as jest.MockedFunction<typeof getFileStream>;+const mockedIsConfigured = isConfiguredProviderMediaType as jest.MockedFunction<+ typeof isConfiguredProviderMediaType+>;++const createMockFile = (type = 'video/mp4'): IMongoFile =>+ ({+ _id: new Types.ObjectId(),+ file_id: 'video-1',+ filename: 'clip.mp4',+ filepath: '/uploads/clip.mp4',+ type,+ bytes: 1024,+ source: 'local',+ user: 'user-1',+ object: 'file',+ usage: 0,+ createdAt: new Date(),+ updatedAt: new Date(),+ }) as unknown as IMongoFile;++const req = { config: {} } as unknown as ServerRequest;+const getStrategyFunctions = jest.fn();++describe('encodeAndFormatVideos - provider formatting', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ mockedValidateVideo.mockResolvedValue({ isValid: true });+ mockedIsConfigured.mockReturnValue(false);+ const file = createMockFile();+ mockedGetFileStream.mockResolvedValue({+ file,+ content: 'AAAA',+ metadata: {+ file_id: file.file_id,+ filepath: file.filepath,+ source: file.source,+ filename: file.filename,+ type: file.type,+ },+ });+ });++ it('emits a Google media block for google', async () => {+ const result = await encodeAndFormatVideos(+ req,+ [createMockFile()],+ { provider: Providers.GOOGLE },+ getStrategyFunctions,+ );+ expect(result.videos).toEqual([{ type: 'media', mimeType: 'video/mp4', data: 'AAAA' }]);+ });++ it('emits an OpenAI-compatible video_url block for openrouter without configuration', async () => {+ const result = await encodeAndFormatVideos(+ req,+ [createMockFile()],+ { provider: Providers.OPENROUTER },+ getStrategyFunctions,+ );+ expect(mockedIsConfigured).not.toHaveBeenCalled();+ expect(result.videos).toEqual([+ { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } },+ ]);+ });++ it('drops the video for a custom endpoint that is not configured for video', async () => {+ const result = await encodeAndFormatVideos(+ req,+ [createMockFile()],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ );+ expect(mockedIsConfigured).toHaveBeenCalledWith(+ req,+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ 'video/mp4',+ );+ expect(result.videos).toEqual([]);+ expect(result.files).toHaveLength(1);+ });++ it('emits a video_url block for a custom endpoint whose config allows video', async () => {+ mockedIsConfigured.mockReturnValue(true);+ const result = await encodeAndFormatVideos(+ req,+ [createMockFile()],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ getStrategyFunctions,+ );+ expect(result.videos).toEqual([+ { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAA' } },+ ]);+ });++ it('never emits a block for providers without a video path, even when configured', async () => {+ mockedIsConfigured.mockReturnValue(true);+ const result = await encodeAndFormatVideos(+ req,+ [createMockFile()],+ { provider: Providers.ANTHROPIC },+ getStrategyFunctions,+ );+ expect(result.videos).toEqual([]);+ });+});diff --git a/packages/api/src/files/encode/video.ts b/packages/api/src/files/encode/video.tsindex 52a9331c8d..2715b36eed 100644--- a/packages/api/src/files/encode/video.ts+++ b/packages/api/src/files/encode/video.ts@@ -1,11 +1,12 @@import { Providers } from '@librechat/agents';-import { isDocumentSupportedProvider } from 'librechat-data-provider';+import { isDocumentSupportedProvider, isOpenAILikeProvider } from 'librechat-data-provider';import type { IMongoFile } from '@librechat/data-schemas';import type { ServerRequest, StrategyFunctions, VideoResult } from '~/types';import {getFileStream,getConfiguredFileSizeLimit,isAttachmentObjectNotFoundError,+ isConfiguredProviderMediaType,} from './utils';import { validateVideo } from '~/files/validation';import { runGuardedEncode } from './memoryGuard';@@ -91,7 +92,14 @@ export async function encodeAndFormatVideos(mimeType: file.type,data: content,});- } else if (provider === Providers.OPENROUTER) {+ } else if (+ provider === Providers.OPENROUTER ||+ (isOpenAILikeProvider(provider) &&+ isConfiguredProviderMediaType(req, { provider, endpoint }, file.type))+ ) {+ /** OpenAI-compatible `video_url` part (OpenRouter, vLLM, and other gateways).+ * Custom endpoints only get here when the admin listed video types in the+ * endpoint's `supportedMimeTypes`, since not every gateway accepts video. */result.videos.push({type: 'video_url',video_url: {diff --git a/packages/api/src/files/upload/errors.ts b/packages/api/src/files/upload/errors.tsnew file mode 100644index 0000000000..11e579abb8--- /dev/null+++ b/packages/api/src/files/upload/errors.ts@@ -0,0 +1,9 @@+/** A direct audio attachment must be representable before the upload is persisted. */+export class UnsupportedProviderAudioError extends Error {+ readonly userErrorStatusCode = 415;++ constructor() {+ super('com_error_files_provider_audio_format');+ this.name = 'UnsupportedProviderAudioError';+ }+}diff --git a/packages/api/src/files/upload/routing.spec.ts b/packages/api/src/files/upload/routing.spec.tsindex 2d8de0dc8a..71ae6289ff 100644--- a/packages/api/src/files/upload/routing.spec.ts+++ b/packages/api/src/files/upload/routing.spec.ts@@ -1,6 +1,10 @@-import { EModelEndpoint } from 'librechat-data-provider';+import { Readable } from 'node:stream';+import { EModelEndpoint, EToolResources, Providers } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';import type { ServerRequest } from '~/types';import { resolveUploadEndpoint, resolveEffectiveToolResource } from './routing';+import { encodeAndFormatAudios } from '~/files/encode/audio';+import { UnsupportedProviderAudioError } from './errors';describe('resolveUploadEndpoint', () => {const req = { user: { id: 'user-1' } } as unknown as ServerRequest;@@ -104,3 +108,141 @@ describe('resolveEffectiveToolResource Responses handling', () => {expect(usable).toBe('context');});});++describe('direct provider audio preflight', () => {+ const makeReq = (legacyFileUploadUX = false, mimetype = 'audio/wma', originalname = 'clip.wma') =>+ Object.assign({} as ServerRequest, {+ file: {+ mimetype,+ originalname,+ fieldname: 'file',+ encoding: '7bit',+ size: 16,+ destination: '/uploads',+ filename: originalname,+ path: '/uploads/audio-1',+ stream: Readable.from(Buffer.alloc(16)),+ buffer: Buffer.alloc(16),+ },+ config: {+ fileConfig: {+ endpoints: { MyGateway: { supportedMimeTypes: ['audio/.*'], legacyFileUploadUX } },+ },+ },+ });++ it.each([false, true])(+ 'rejects unsupported audio before processing (legacy=%s)',+ async (legacy) => {+ await expect(+ resolveEffectiveToolResource({+ req: makeReq(legacy),+ metadata: { endpoint: 'MyGateway' },+ getAgent: jest.fn(),+ }),+ ).rejects.toMatchObject({+ message: 'com_error_files_provider_audio_format',+ userErrorStatusCode: 415,+ });+ },+ );++ it('uses the resolved agent endpoint and reuses the agent read', async () => {+ const req = makeReq();+ const getAgent = jest.fn().mockResolvedValue({ provider: 'MyGateway' });+ const metadata = { endpoint: EModelEndpoint.agents, agent_id: 'agent_saved01' };+ await resolveUploadEndpoint({ req, metadata, getAgent });+ await expect(resolveEffectiveToolResource({ req, metadata, getAgent })).rejects.toBeInstanceOf(+ UnsupportedProviderAudioError,+ );+ expect(getAgent).toHaveBeenCalledTimes(1);+ });++ it.each([EToolResources.context, EToolResources.ocr, EToolResources.execute_code])(+ 'preserves explicit %s uploads',+ async (tool_resource) => {+ await expect(+ resolveEffectiveToolResource({+ req: makeReq(true),+ metadata: { endpoint: 'MyGateway', tool_resource },+ getAgent: jest.fn(),+ }),+ ).resolves.toBe(+ tool_resource === EToolResources.ocr ? EToolResources.context : tool_resource,+ );+ },+ );++ it('preserves audio explicitly routed to transcription', async () => {+ const req = makeReq();+ req.config!.fileConfig!.defaultLLMDeliveryPath = { overrides: { 'audio/*': 'text' } };+ await expect(+ resolveEffectiveToolResource({+ req,+ metadata: { endpoint: 'MyGateway' },+ getAgent: jest.fn(),+ }),+ ).resolves.toBe(EToolResources.context);+ });++ it.each([EModelEndpoint.google, Providers.VERTEXAI])(+ 'does not apply input_audio rules to %s',+ async (endpoint) => {+ await expect(+ resolveEffectiveToolResource({+ req: makeReq(),+ metadata: { endpoint },+ getAgent: jest.fn(),+ }),+ ).resolves.toBeUndefined();+ },+ );++ it('also rejects unsupported OpenRouter audio', async () => {+ await expect(+ resolveEffectiveToolResource({+ req: makeReq(),+ metadata: { endpoint: Providers.OPENROUTER },+ getAgent: jest.fn(),+ }),+ ).rejects.toBeInstanceOf(UnsupportedProviderAudioError);+ });++ it.each([+ ['audio/wave', 'clip.wave', 'wav'],+ ['audio/mpeg', 'recording', 'mp3'],+ ['audio/unknown', 'clip.pcm16', 'pcm16'],+ ])(+ 'delivers an accepted %s upload through the real encoder',+ async (mimetype, originalname, format) => {+ const req = makeReq(false, mimetype, originalname);+ await expect(+ resolveEffectiveToolResource({+ req,+ metadata: { endpoint: 'MyGateway' },+ getAgent: jest.fn(),+ }),+ ).resolves.toBeUndefined();+ const file = {+ file_id: 'audio-1',+ filename: originalname,+ filepath: '/uploads/audio-1',+ type: mimetype,+ bytes: 16,+ source: 'local',+ } as IMongoFile;+ const result = await encodeAndFormatAudios(+ req,+ [file],+ { provider: Providers.OPENAI, endpoint: 'MyGateway' },+ () => ({ getDownloadStream: async () => Readable.from(Buffer.alloc(16)) }),+ );+ expect(result.audios).toEqual([+ {+ type: 'input_audio',+ input_audio: { data: Buffer.alloc(16).toString('base64'), format },+ },+ ]);+ },+ );+});diff --git a/packages/api/src/files/upload/routing.ts b/packages/api/src/files/upload/routing.tsindex 31f571cbae..f6a5e1d7a7 100644--- a/packages/api/src/files/upload/routing.ts+++ b/packages/api/src/files/upload/routing.ts@@ -1,13 +1,21 @@import {+ Providers,+ EModelEndpoint,EToolResources,mergeFileConfig,+ isExplicitMimeConfig,+ isOpenAILikeProvider,+ isKnownProviderIdentifier,isAssistantsEndpoint,isResponsesApiUpload,getEndpointFileConfig,resolveUploadLLMDeliveryPath,isSpeechProviderConfigured,+ getCustomEndpointProvider,} from 'librechat-data-provider';import type { ServerRequest } from '~/types';+import { UnsupportedProviderAudioError } from './errors';+import { getAudioFormat } from '~/files/encode/utils';/** The subset of an agent record this module reads. */export interface UploadAgent {@@ -27,6 +35,7 @@ export interface UploadMetadata {/** The uploaded file fields routing depends on. */export interface UploadFile {mimetype: string;+ originalname?: string;}export type GetUploadAgent = (params: { id: string }) => Promise<UploadAgent | null>;@@ -112,14 +121,35 @@ export async function resolveEffectiveToolResource({const fileConfig = mergeFileConfig(req.config?.fileConfig);const endpoint = await resolveUploadEndpoint({ req, metadata, getAgent });const endpointConfig = getEndpointFileConfig({ fileConfig, endpoint });+ const endpointProvider = getCustomEndpointProvider(req.config?.endpoints?.custom, endpoint);+ const file = req.file as UploadFile;const path = resolveUploadLLMDeliveryPath({toolResource: metadata.tool_resource,- mimeType: (req.file as UploadFile).mimetype,+ mimeType: file.mimetype,endpointConfig,fileConfig,endpoint,+ endpointProvider,useResponsesApi: isResponsesApiUpload(metadata.useResponsesApi),sttConfigured: isSpeechProviderConfigured(req.config?.speech?.stt),});+ const provider =+ endpointProvider ?? (isKnownProviderIdentifier(endpoint) ? endpoint : EModelEndpoint.openAI);+ const usesInputAudio =+ provider?.toLowerCase() === Providers.OPENROUTER ||+ (isOpenAILikeProvider(provider) &&+ isExplicitMimeConfig(endpointConfig.supportedMimeTypes) &&+ fileConfig.checkType?.(file.mimetype, endpointConfig.supportedMimeTypes));+ if (+ path === 'provider' &&+ endpoint != null &&+ endpoint !== EModelEndpoint.agents &&+ !isAssistantsEndpoint(endpoint) &&+ file.mimetype.startsWith('audio/') &&+ usesInputAudio &&+ !getAudioFormat(file.mimetype, file.originalname ?? '')+ ) {+ throw new UnsupportedProviderAudioError();+ }return path === 'text' ? EToolResources.context : undefined;}diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.tsindex b49a1a1f90..4d606f2482 100644--- a/packages/api/src/types/files.ts+++ b/packages/api/src/types/files.ts@@ -39,7 +39,7 @@ export interface GoogleVideoBlock {data: string;}-/** OpenRouter video block format */+/** OpenAI-compatible video block format (OpenRouter, vLLM, configured custom endpoints) */export interface OpenRouterVideoBlock {type: 'video_url';video_url: {@@ -136,7 +136,7 @@ export interface GoogleAudioBlock {data: string;}-/** OpenRouter audio block format */+/** OpenAI-compatible audio block format (OpenRouter, vLLM, configured custom endpoints) */export interface OpenRouterAudioBlock {type: 'input_audio';input_audio: {diff --git a/packages/api/src/utils/files.spec.ts b/packages/api/src/utils/files.spec.tsindex f8910b37c4..59eb11decc 100644--- a/packages/api/src/utils/files.spec.ts+++ b/packages/api/src/utils/files.spec.ts@@ -4,6 +4,7 @@ import {flattenArtifactPath,resolveUploadErrorMessage,} from './files';+import { UnsupportedProviderAudioError } from '~/files/upload/errors';jest.mock('node:crypto', () => {const actualModule = jest.requireActual('node:crypto');@@ -503,6 +504,14 @@ describe('flattenArtifactPath', () => {});describe('resolveUploadErrorMessage', () => {+ it.each([false, true])(+ 'preserves the localized audio preflight error (redaction=%s)',+ (redact) => {+ expect(+ resolveUploadErrorMessage(new UnsupportedProviderAudioError(), undefined, redact),+ ).toBe('com_error_files_provider_audio_format');+ },+ );test('returns default message for null error', () => {expect(resolveUploadErrorMessage(null)).toBe('Error processing file');});diff --git a/packages/api/src/utils/files.ts b/packages/api/src/utils/files.tsindex b313dc9b6f..5be59b27aa 100644--- a/packages/api/src/utils/files.ts+++ b/packages/api/src/utils/files.ts@@ -2,6 +2,7 @@ import path from 'path';import crypto from 'node:crypto';import { createReadStream } from 'fs';import { readFile, stat } from 'fs/promises';+import { UnsupportedProviderAudioError } from '~/files/upload/errors';const USER_FACING_UPLOAD_ERRORS = [['Invalid file format', 'Invalid file format'],@@ -98,6 +99,9 @@ export function resolveUploadErrorMessage(defaultMessage = 'Error processing file',redactDetails = false,): string {+ if (error instanceof UnsupportedProviderAudioError) {+ return error.message;+ }const errorMessage = error?.message;if (!errorMessage) {return defaultMessage;diff --git a/packages/data-provider/src/file-config.spec.ts b/packages/data-provider/src/file-config.spec.tsindex 1ee9e035cc..164e75fb75 100644--- a/packages/data-provider/src/file-config.spec.ts+++ b/packages/data-provider/src/file-config.spec.ts@@ -9,6 +9,7 @@ import {bedrockDocumentMimeTypes,isAnthropicDocumentType,isPermissiveMimeConfig,+ isExplicitMimeConfig,convertStringsToRegex,setFileConfigRegexCompiler,documentParserMimeTypes,@@ -1399,6 +1400,35 @@ describe('getEndpointFileConfig', () => {});});+describe('isExplicitMimeConfig', () => {+ it('is false for undefined or empty lists', () => {+ expect(isExplicitMimeConfig(undefined)).toBe(false);+ expect(isExplicitMimeConfig([])).toBe(false);+ });++ it('is false for the built-in default list (inherited, not configured)', () => {+ expect(isExplicitMimeConfig(supportedMimeTypes)).toBe(false);+ const endpointConfig = getEndpointFileConfig({+ fileConfig: mergeFileConfig({ endpoints: { Other: { fileLimit: 1 } } }),+ endpoint: 'MyGateway',+ endpointType: 'custom',+ });+ expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(false);+ });++ it('is true for an admin-configured list, permissive or not', () => {+ const endpointConfig = getEndpointFileConfig({+ fileConfig: mergeFileConfig({+ endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } },+ }),+ endpoint: 'MyGateway',+ endpointType: 'custom',+ });+ expect(isExplicitMimeConfig(endpointConfig.supportedMimeTypes)).toBe(true);+ expect(isExplicitMimeConfig([/.*/])).toBe(true);+ });+});+describe('isPermissiveMimeConfig', () => {it('returns true for wildcard .* pattern', () => {expect(isPermissiveMimeConfig([/.*/])).toBe(true);diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.tsindex b3f55413b9..0890dd24ae 100644--- a/packages/data-provider/src/file-config.ts+++ b/packages/data-provider/src/file-config.ts@@ -674,6 +674,18 @@ export const isPermissiveMimeConfig = (types?: RegexLike[]): boolean => {return types.some((regex) => regex.test('x-librechat/x-probe'));};+/**+ * Detects whether an endpoint's `supportedMimeTypes` were set by the admin rather than inherited+ * from the built-in default list. Inheritance is signaled by referential identity with+ * `supportedMimeTypes`, which `mergeWithDefault` preserves for unconfigured endpoints.+ */+export const isExplicitMimeConfig = (types?: RegexLike[]): types is RegexLike[] => {+ if (!types || types.length === 0) {+ return false;+ }+ return types !== supportedMimeTypes;+};+/** The kind of content a provider upload path can actually send to the model. */export type MimeUploadCategory = 'image' | 'document' | 'audio' | 'video';diff --git a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts b/packages/data-provider/src/resolve-llm-delivery-path.spec.tsindex 520e60a63f..b62c3a26f9 100644--- a/packages/data-provider/src/resolve-llm-delivery-path.spec.ts+++ b/packages/data-provider/src/resolve-llm-delivery-path.spec.ts@@ -1,11 +1,15 @@import type { TDefaultLLMDeliveryPathConfig } from './file-config';+import type { TEndpoint } from './config';import {isNativelyReadableText,canToolResourceConsume,resolveUploadDestination,+ getCustomEndpointProvider,resolveDefaultLLMDeliveryPath,+ resolveUploadLLMDeliveryPath,SYSTEM_LLM_DELIVERY_DEFAULTS,} from './resolve-llm-delivery-path';+import { mergeFileConfig, supportedMimeTypes, getEndpointFileConfig } from './file-config';describe('resolveDefaultLLMDeliveryPath', () => {it('should return system default for images when no config provided', () => {@@ -300,6 +304,97 @@ describe('resolveDefaultLLMDeliveryPath', () => {);});+ describe('media a custom endpoint opted into', () => {+ /* The encoders emit OpenAI-format media parts for an OpenAI-compatible endpoint only+ * when the admin listed the type in its `supportedMimeTypes`, so the route has to+ * agree: an explicit match is provider-capable, the inherited default list is not. */+ const explicit = [/^image\/.*$/, /^application\/pdf$/, /^video\/.*$/, /^audio\/wav$/];+ const resolve = (mimeType: string, endpoint: string, types?: RegExp[]) =>+ resolveDefaultLLMDeliveryPath(+ mimeType,+ undefined,+ undefined,+ endpoint,+ undefined,+ true,+ types,+ );++ it('keeps an explicitly allowed type on the provider path for a custom endpoint', () => {+ expect(resolve('video/mp4', 'MyGateway', explicit)).toBe('provider');+ expect(resolve('audio/wav', 'MyGateway', explicit)).toBe('provider');+ });++ it('still downgrades a media type the allowlist does not name', () => {+ expect(resolve('audio/mpeg', 'MyGateway', explicit)).toBe('text');+ });++ it('does not read the inherited default list as an opt-in', () => {+ expect(resolve('video/mp4', 'MyGateway', supportedMimeTypes)).toBe('none');+ expect(resolve('video/mp4', 'MyGateway', [])).toBe('none');+ });++ it('does not opt in a built-in endpoint, which the client offers no media for', () => {+ /* Anthropic and Bedrock encoders have no media branch at all, and OpenAI/Azure are+ * left out because the picker and drag-drop only open media for custom endpoints:+ * a route the client cannot send to is a capability with no entry point. */+ expect(resolve('video/mp4', 'openAI', explicit)).toBe('none');+ expect(resolve('video/mp4', 'azureOpenAI', explicit)).toBe('none');+ expect(resolve('video/mp4', 'anthropic', explicit)).toBe('none');+ expect(resolve('video/mp4', 'bedrock', explicit)).toBe('none');+ });++ it('keeps a custom endpoint that runs as Anthropic on its previous route', () => {+ /* A custom endpoint may declare `provider: anthropic`, and the encoders emit+ * OpenAI-format parts only, so the opt-in would deliver nothing there. Audio keeps+ * its transcription route and video stays off the model path. */+ const endpointConfig = { supportedMimeTypes: explicit };+ const anthropic = { mimeType: 'video/mp4', endpointConfig, endpoint: 'MyClaude' };+ expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'anthropic' })).toBe(+ 'none',+ );+ expect(+ resolveUploadLLMDeliveryPath({+ ...anthropic,+ mimeType: 'audio/wav',+ endpointProvider: 'anthropic',+ sttConfigured: true,+ }),+ ).toBe('text');+ expect(resolveUploadLLMDeliveryPath({ ...anthropic, endpointProvider: 'openAI' })).toBe(+ 'provider',+ );+ expect(resolveUploadLLMDeliveryPath(anthropic)).toBe('provider');+ });++ it('reaches the upload resolver through the merged endpoint config', () => {+ /* The real merge, so the identity check that separates a configured list from the+ * inherited default is exercised the way the upload route exercises it. */+ const fileConfig = mergeFileConfig({+ endpoints: { MyGateway: { supportedMimeTypes: ['image/.*', 'video/.*'] } },+ });+ const configured = getEndpointFileConfig({ fileConfig, endpoint: 'MyGateway' });+ const inherited = getEndpointFileConfig({ fileConfig, endpoint: 'OtherGateway' });++ expect(+ resolveUploadLLMDeliveryPath({+ mimeType: 'video/mp4',+ endpointConfig: configured,+ fileConfig,+ endpoint: 'MyGateway',+ }),+ ).toBe('provider');+ expect(+ resolveUploadLLMDeliveryPath({+ mimeType: 'video/mp4',+ endpointConfig: inherited,+ fileConfig,+ endpoint: 'OtherGateway',+ }),+ ).toBe('none');+ });+ });+it('leaves media alone when no endpoint is resolved at all', () => {/* An ephemeral agent reports no usable endpoint, which is not the same as naming one* we cannot identify. */@@ -545,6 +640,27 @@ describe('resolveUploadDestination', () => {});});+describe('getCustomEndpointProvider', () => {+ const custom = [+ { name: 'My Claude', provider: 'anthropic' },+ { name: 'Ollama', provider: 'anthropic' },+ { name: 'MyGateway' },+ ] as Array<Partial<Pick<TEndpoint, 'name' | 'provider'>>>;++ it('returns the declared dialect for a custom endpoint, matching the normalized name', () => {+ expect(getCustomEndpointProvider(custom, 'My Claude')).toBe('anthropic');+ /* The same normalization the file config lookup applies to endpoint names. */+ expect(getCustomEndpointProvider(custom, 'ollama')).toBe('anthropic');+ });++ it('returns nothing for an endpoint without a dialect, an unknown one, or no config', () => {+ expect(getCustomEndpointProvider(custom, 'MyGateway')).toBeUndefined();+ expect(getCustomEndpointProvider(custom, 'Other')).toBeUndefined();+ expect(getCustomEndpointProvider(undefined, 'My Claude')).toBeUndefined();+ expect(getCustomEndpointProvider(custom, undefined)).toBeUndefined();+ });+});+describe('isNativelyReadableText', () => {it('admits the application types whose payload is text', () => {/* Kept in step with the textual set in the content-protection code. Missing one sendsdiff --git a/packages/data-provider/src/resolve-llm-delivery-path.ts b/packages/data-provider/src/resolve-llm-delivery-path.tsindex bb6148efaf..bccf7f566c 100644--- a/packages/data-provider/src/resolve-llm-delivery-path.ts+++ b/packages/data-provider/src/resolve-llm-delivery-path.ts@@ -1,13 +1,57 @@import type { TDefaultLLMDeliveryPath, TDefaultLLMDeliveryPathConfig } from './file-config';-import type { EndpointFileConfig, FileConfig } from './types/files';+import type { EndpointFileConfig, FileConfig, RegexLike } from './types/files';+import type { TEndpoint } from './config';+import {+ retrievalMimeTypes,+ isExplicitMimeConfig,+ isBedrockDocumentType,+ codeInterpreterMimeTypes,+ fileConfig as baseFileConfig,+} from './file-config';import {EModelEndpoint,- isDocumentSupportedProvider,+ isOpenAILikeProvider,isKnownProviderIdentifier,isMediaSupportedProvider,+ isDocumentSupportedProvider,} from './schemas';-import { retrievalMimeTypes, isBedrockDocumentType, codeInterpreterMimeTypes } from './file-config';import { EToolResources } from './types/assistants';+import { normalizeEndpointName } from './utils';++/**+ * The native provider a custom endpoint declares, when it declares one. A custom endpoint+ * speaks OpenAI's API unless its config names another dialect, and the upload route needs+ * that answer for the same reason request initialization does: the media encoders emit+ * OpenAI-format parts, so a custom endpoint running as Anthropic receives none.+ */+export function getCustomEndpointProvider(+ customEndpoints: Array<Partial<Pick<TEndpoint, 'name' | 'provider'>>> | undefined,+ endpoint?: string | null,+): string | undefined {+ if (!customEndpoints || !endpoint) {+ return undefined;+ }+ const normalized = normalizeEndpointName(endpoint);+ return customEndpoints.find((config) => normalizeEndpointName(config.name ?? '') === normalized)+ ?.provider;+}++/** A custom endpoint emits OpenAI-format media parts only for the types the admin listed+ * in its `supportedMimeTypes`; the inherited default list is not an opt-in. A name that+ * is not a known provider is a custom endpoint. Mirrors `isConfiguredProviderMediaType`+ * on the encoder side, so the route and the encoder agree on which uploads the provider+ * actually receives; the built-in endpoints are left out because the client offers no+ * media for them. */+const isConfiguredMediaEndpoint = (+ mimeType: string,+ endpoint: string,+ supportedMimeTypes?: RegexLike[],+): boolean => {+ if (!isExplicitMimeConfig(supportedMimeTypes) || isKnownProviderIdentifier(endpoint)) {+ return false;+ }+ return baseFileConfig.checkType(mimeType, supportedMimeTypes);+};/** Audio and video reach the model only through the media encoders, which support a* narrower provider set than documents. Images use the broadly supported vision@@ -16,9 +60,13 @@ const isProviderCapable = (mimeType: string,endpoint: string,useResponsesApi?: boolean,+ supportedMimeTypes?: RegexLike[],): boolean => {if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) {- return isMediaSupportedProvider(endpoint);+ return (+ isMediaSupportedProvider(endpoint) ||+ isConfiguredMediaEndpoint(mimeType, endpoint, supportedMimeTypes)+ );}if (mimeType === 'application/pdf') {/* Azure is out of the document set because it needs the Responses API for native@@ -104,6 +152,7 @@ export function resolveDefaultLLMDeliveryPath(endpoint?: string,useResponsesApi?: boolean,sttConfigured?: boolean,+ supportedMimeTypes?: RegexLike[],): TDefaultLLMDeliveryPath {const wildcard = mimeType.split('/')[0] + '/*';@@ -149,10 +198,11 @@ export function resolveDefaultLLMDeliveryPath(const namedEndpoint = endpoint != null && endpoint !== EModelEndpoint.agents;const providerKnown = namedEndpoint && isKnownProviderIdentifier(endpoint);/* Media is judged for any named endpoint, identified or not. The media encoders emit a- * payload only for the providers they name, so a custom endpoint gets nothing whatever- * it proxies to, and leaving it on the provider path delivers neither media nor text.- * Documents keep the narrower rule: an unidentified endpoint is usually OpenAI- or- * Anthropic-compatible, both of which do carry them. */+ * payload only for the providers they name, or for an OpenAI-compatible endpoint whose+ * admin listed the type in its `supportedMimeTypes`; any other custom endpoint gets+ * nothing whatever it proxies to, and leaving it on the provider path delivers neither+ * media nor text. Documents keep the narrower rule: an unidentified endpoint is usually+ * OpenAI- or Anthropic-compatible, both of which do carry them. */const isMedia = mimeType.startsWith('audio/') || mimeType.startsWith('video/');/* Audio's text path is transcription, so on a deployment with no speech provider it is* not recoverable at all. Routing it to text there sends the upload to a service that@@ -164,7 +214,7 @@ export function resolveDefaultLLMDeliveryPath(if (systemDefault === 'provider' &&canJudgeCapability &&- !isProviderCapable(mimeType, endpoint as string, useResponsesApi)+ !isProviderCapable(mimeType, endpoint as string, useResponsesApi, supportedMimeTypes)) {/* Downgrading is only useful where text can actually be recovered. Video has no* extraction step: speech-to-text covers audio, and the default text matcher accepts@@ -206,6 +256,7 @@ export function resolveDefaultUploadLLMDeliveryPath({endpointConfig,fileConfig,endpoint,+ endpointProvider,useResponsesApi,sttConfigured,}: {@@ -213,12 +264,19 @@ export function resolveDefaultUploadLLMDeliveryPath({endpointConfig?: EndpointFileConfig;fileConfig?: FileConfig;endpoint?: string;+ /** The provider the endpoint runs as, when the caller knows it: a custom endpoint's+ * declared dialect at upload time, the agent's resolved provider at turn time. */+ endpointProvider?: string | null;useResponsesApi?: boolean;sttConfigured?: boolean;}): TDefaultLLMDeliveryPath {if (endpointConfig?.legacyFileUploadUX === true) {return 'provider';}+ /* The media opt-in exists for OpenAI-format parts, so an endpoint known to run as+ * something else — a custom endpoint declaring `provider: anthropic` — keeps the+ * capability gate it had, where audio still reaches transcription. */+ const runsAsOpenAI = endpointProvider == null || isOpenAILikeProvider(endpointProvider);return resolveDefaultLLMDeliveryPath(mimeType,endpointConfig?.defaultLLMDeliveryPath,@@ -226,6 +284,7 @@ export function resolveDefaultUploadLLMDeliveryPath({endpoint,useResponsesApi,sttConfigured,+ runsAsOpenAI ? endpointConfig?.supportedMimeTypes : undefined,);}@@ -237,6 +296,7 @@ export function resolveUploadLLMDeliveryPath({fileConfig,endpoint,useResponsesApi,+ endpointProvider,sttConfigured,}: {toolResource?: string | null;@@ -244,6 +304,7 @@ export function resolveUploadLLMDeliveryPath({endpointConfig?: EndpointFileConfig;fileConfig?: FileConfig;endpoint?: string;+ endpointProvider?: string | null;useResponsesApi?: boolean;sttConfigured?: boolean;}): TDefaultLLMDeliveryPath {@@ -258,6 +319,7 @@ export function resolveUploadLLMDeliveryPath({endpointConfig,fileConfig,endpoint,+ endpointProvider,useResponsesApi,sttConfigured,});