Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📎 fix: Route Unrecognized File Types via supportedMimeTypes Config (#12508)

Danny Avila6ecd1b510faa
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js
index cdce2edf4d..905cadfd23 100644
--- a/api/app/clients/BaseClient.js
+++ b/api/app/clients/BaseClient.js
@@ -17,11 +17,13 @@ const {
ContentTypes,
excludedKeys,
EModelEndpoint,
+ mergeFileConfig,
isParamEndpoint,
isAgentsEndpoint,
isEphemeralAgentId,
supportsBalanceCheck,
isBedrockDocumentType,
+ getEndpointFileConfig,
} = require('librechat-data-provider');
const { getStrategyFunctions } = require('~/server/services/Files/strategies');
const { logViolation } = require('~/cache');
@@ -71,6 +73,10 @@ class BaseClient {
this.currentMessages = [];
/** @type {import('librechat-data-provider').VisionModes | undefined} */
this.visionMode;
+ /** @type {import('librechat-data-provider').FileConfig | undefined} */
+ this._mergedFileConfig;
+ /** @type {import('librechat-data-provider').EndpointFileConfig | undefined} */
+ this._endpointFileConfig;
}
setOptions() {
@@ -1160,6 +1166,16 @@ class BaseClient {
const provider = this.options.agent?.provider ?? this.options.endpoint;
const isBedrock = provider === EModelEndpoint.bedrock;
+ if (!this._mergedFileConfig && this.options.req?.config?.fileConfig) {
+ this._mergedFileConfig = mergeFileConfig(this.options.req.config.fileConfig);
+ const endpoint = this.options.agent?.endpoint ?? this.options.endpoint;
+ this._endpointFileConfig = getEndpointFileConfig({
+ fileConfig: this._mergedFileConfig,
+ endpoint,
+ endpointType: this.options.endpointType,
+ });
+ }
+
for (const file of attachments) {
/** @type {FileSources} */
const source = file.source ?? FileSources.local;
@@ -1186,6 +1202,14 @@ class BaseClient {
} else if (file.type.startsWith('audio/')) {
categorizedAttachments.audios.push(file);
allFiles.push(file);
+ } else if (
+ file.type &&
+ this._mergedFileConfig &&
+ this._endpointFileConfig?.supportedMimeTypes &&
+ this._mergedFileConfig.checkType(file.type, this._endpointFileConfig.supportedMimeTypes)
+ ) {
+ categorizedAttachments.documents.push(file);
+ allFiles.push(file);
}
}
diff --git a/packages/api/src/files/encode/document.spec.ts b/packages/api/src/files/encode/document.spec.ts
index 2b8e231048..f285d47d5d 100644
--- a/packages/api/src/files/encode/document.spec.ts
+++ b/packages/api/src/files/encode/document.spec.ts
@@ -56,13 +56,16 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
});
/** Helper to create a mock request with file config */
- const createMockRequest = (fileSizeLimit?: number): Partial<AppConfig> => ({
+ const createMockRequest = (
+ fileSizeLimit?: number,
+ provider: string = Providers.OPENAI,
+ ): Partial<AppConfig> => ({
config:
fileSizeLimit !== undefined
? {
fileConfig: {
endpoints: {
- [Providers.OPENAI]: {
+ [provider]: {
fileSizeLimit,
},
},
@@ -747,4 +750,235 @@ describe('encodeAndFormatDocuments - fileConfig integration', () => {
});
});
});
+
+ describe('Generic document encoding path', () => {
+ it('should format text/plain for Anthropic with citations enabled', async () => {
+ const req = createMockRequest(30) as ServerRequest;
+ const file = createMockDocFile(1, 'text/plain', 'notes.txt');
+
+ const mockContent = Buffer.from('plain text content').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.ANTHROPIC },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).toMatchObject({
+ type: 'document',
+ source: {
+ type: 'base64',
+ media_type: 'text/plain',
+ data: mockContent,
+ },
+ citations: { enabled: true },
+ context: 'File: "notes.txt"',
+ });
+ expect(result.files).toHaveLength(1);
+ });
+
+ it('should format text/html for Anthropic with citations enabled', async () => {
+ const req = createMockRequest(30) as ServerRequest;
+ const file = createMockDocFile(1, 'text/html', 'page.html');
+
+ const mockContent = Buffer.from('<html>content</html>').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.ANTHROPIC },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).toMatchObject({
+ type: 'document',
+ source: { type: 'base64', media_type: 'text/html', data: mockContent },
+ citations: { enabled: true },
+ });
+ });
+
+ it('should format application/json for Anthropic without citations', async () => {
+ const req = createMockRequest(30) as ServerRequest;
+ const file = createMockDocFile(1, 'application/json', 'data.json');
+
+ const mockContent = Buffer.from('{"key":"value"}').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.ANTHROPIC },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).not.toHaveProperty('citations');
+ });
+
+ it('should format text/csv for OpenAI responses API', async () => {
+ const req = createMockRequest(15) as ServerRequest;
+ const file = createMockDocFile(1, 'text/csv', 'data.csv');
+
+ const mockContent = Buffer.from('a,b\n1,2').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.OPENAI, useResponsesApi: true },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).toMatchObject({
+ type: 'input_file',
+ filename: 'data.csv',
+ file_data: `data:text/csv;base64,${mockContent}`,
+ });
+ expect(result.files).toHaveLength(1);
+ });
+
+ it('should format XLSX for Google/VertexAI as media block', async () => {
+ const req = createMockRequest(25) as ServerRequest;
+ const mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
+ const file = createMockDocFile(2, mimeType, 'report.xlsx');
+
+ const mockContent = Buffer.from('xlsx-binary').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.GOOGLE },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).toMatchObject({
+ type: 'media',
+ mimeType,
+ data: mockContent,
+ });
+ expect(result.files).toHaveLength(1);
+ });
+
+ it('should format text/plain for standard OpenAI-like provider as file block', async () => {
+ const req = createMockRequest(15) as ServerRequest;
+ const file = createMockDocFile(1, 'text/plain', 'readme.txt');
+
+ const mockContent = Buffer.from('readme content').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.OPENAI },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(1);
+ expect(result.documents[0]).toMatchObject({
+ type: 'file',
+ file: {
+ filename: 'readme.txt',
+ file_data: `data:text/plain;base64,${mockContent}`,
+ },
+ });
+ expect(result.files).toHaveLength(1);
+ });
+
+ it('should skip non-Bedrock-document types for Bedrock provider', async () => {
+ const req = createMockRequest() as ServerRequest;
+ const file = createMockDocFile(1, 'application/zip', 'archive.zip');
+
+ const mockContent = Buffer.from('zip-content').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.BEDROCK },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(0);
+ expect(result.files).toHaveLength(0);
+ });
+
+ it('should throw when generic file exceeds configured size limit', async () => {
+ const req = createMockRequest(1, Providers.ANTHROPIC) as ServerRequest;
+ const file = createMockDocFile(2, 'text/plain', 'large.txt');
+
+ const largeContent = Buffer.alloc(2 * 1024 * 1024).toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: largeContent,
+ metadata: file,
+ });
+
+ await expect(
+ encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.ANTHROPIC },
+ mockStrategyFunctions,
+ ),
+ ).rejects.toThrow('File size');
+ });
+
+ it('should not push metadata when provider has no handler', async () => {
+ const req = createMockRequest(15) as ServerRequest;
+ const file = createMockDocFile(1, 'text/plain', 'test.txt');
+
+ const mockContent = Buffer.from('content').toString('base64');
+ mockedGetFileStream.mockResolvedValue({
+ file,
+ content: mockContent,
+ metadata: file,
+ });
+
+ const result = await encodeAndFormatDocuments(
+ req,
+ [file],
+ { provider: Providers.AZURE as Providers },
+ mockStrategyFunctions,
+ );
+
+ expect(result.documents).toHaveLength(0);
+ expect(result.files).toHaveLength(0);
+ });
+ });
});
diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.ts
index 4126a51376..f04ad43ef2 100644
--- a/packages/api/src/files/encode/document.ts
+++ b/packages/api/src/files/encode/document.ts
@@ -7,6 +7,7 @@ import {
} from 'librechat-data-provider';
import type { IMongoFile } from '@librechat/data-schemas';
import type {
+ DocumentBlock,
AnthropicDocumentBlock,
StrategyFunctions,
DocumentResult,
@@ -15,16 +16,85 @@ import type {
import { validatePdf, validateBedrockDocument } from '~/files/validation';
import { getFileStream, getConfiguredFileSizeLimit } from './utils';
+const ANTHROPIC_CITATION_TYPES = new Set([
+ 'application/pdf',
+ 'text/plain',
+ 'text/html',
+ 'text/markdown',
+]);
+
/**
- * Processes and encodes document files for various providers
- * @param req - Express request object
- * @param files - Array of file objects to process
- * @param params - Object containing provider, endpoint, and other options
- * @param params.provider - The provider name
- * @param params.endpoint - Optional endpoint name for file config lookup
- * @param params.useResponsesApi - Whether to use responses API format
- * @param getStrategyFunctions - Function to get strategy functions
- * @returns Promise that resolves to documents and file metadata
+ * Formats a base64-encoded document into the appropriate provider-specific block.
+ * Returns `null` when the provider has no matching handler.
+ */
+function formatDocumentBlock(
+ provider: Providers,
+ mimeType: string,
+ content: string,
+ filename: string | undefined,
+ useResponsesApi: boolean | undefined,
+): DocumentBlock | null {
+ if (provider === Providers.ANTHROPIC) {
+ const document: AnthropicDocumentBlock = {
+ type: 'document',
+ source: {
+ type: 'base64',
+ media_type: mimeType,
+ data: content,
+ },
+ };
+
+ if (ANTHROPIC_CITATION_TYPES.has(mimeType)) {
+ document.citations = { enabled: true };
+ }
+
+ if (filename) {
+ document.context = `File: "${filename}"`;
+ }
+
+ return document;
+ }
+
+ const resolvedFilename = filename ?? 'document';
+
+ if (useResponsesApi) {
+ return {
+ type: 'input_file',
+ filename: resolvedFilename,
+ file_data: `data:${mimeType};base64,${content}`,
+ };
+ }
+
+ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {
+ return {
+ type: 'media',
+ mimeType,
+ data: content,
+ };
+ }
+
+ if (isOpenAILikeProvider(provider) && provider !== Providers.AZURE) {
+ return {
+ type: 'file',
+ file: {
+ filename: resolvedFilename,
+ file_data: `data:${mimeType};base64,${content}`,
+ },
+ };
+ }
+
+ return null;
+}
+
+/**
+ * Encodes and formats document files for various providers.
+ *
+ * Callers are responsible for pre-filtering `files` to types the endpoint accepts
+ * (e.g., via `supportedMimeTypes` in `processAttachments`). This function processes
+ * every file it receives and dispatches to the appropriate provider format:
+ * - **Bedrock**: Only encodes types in `bedrockDocumentFormats`; all others are skipped.
+ * - **PDF**: Validated via `validatePdf` before encoding.
+ * - **Generic types**: Encoded with a provider-specific size check.
*/
export async function encodeAndFormatDocuments(
req: ServerRequest,
@@ -43,25 +113,22 @@ export async function encodeAndFormatDocuments(
const isBedrock = provider === Providers.BEDROCK;
const isDocSupported = isDocumentSupportedProvider(provider);
- const documentFiles = files.filter((file) => {
- if (isBedrock && isBedrockDocumentType(file.type)) {
- return true;
- }
- return file.type === 'application/pdf' || file.type?.startsWith('application/');
- });
+ if (!isDocSupported && !isBedrock) {
+ return result;
+ }
+
+ const processableFiles = isBedrock
+ ? files.filter((file) => isBedrockDocumentType(file.type))
+ : files;
- if (!documentFiles.length) {
+ if (!processableFiles.length) {
return result;
}
+ const configuredFileSizeLimit = getConfiguredFileSizeLimit(req, { provider, endpoint });
+
const results = await Promise.allSettled(
- documentFiles.map((file) => {
- const isProcessable = isBedrock
- ? isBedrockDocumentType(file.type)
- : file.type === 'application/pdf' && isDocSupported;
- if (!isProcessable) {
- return Promise.resolve(null);
- }
+ processableFiles.map((file) => {
return getFileStream(req, file, encodingMethods, getStrategyFunctions);
}),
);
@@ -82,7 +149,6 @@ export async function encodeAndFormatDocuments(
continue;
}
- const configuredFileSizeLimit = getConfiguredFileSizeLimit(req, { provider, endpoint });
const mimeType = file.type ?? '';
if (isBedrock && isBedrockDocumentType(mimeType)) {
@@ -130,44 +196,37 @@ export async function encodeAndFormatDocuments(
throw new Error(`PDF validation failed: ${validation.error}`);
}
- if (provider === Providers.ANTHROPIC) {
- const document: AnthropicDocumentBlock = {
- type: 'document',
- source: {
- type: 'base64',
- media_type: 'application/pdf',
- data: content,
- },
- citations: { enabled: true },
- };
-
- if (file.filename) {
- document.context = `File: "${file.filename}"`;
- }
-
- result.documents.push(document);
- } else if (useResponsesApi) {
- result.documents.push({
- type: 'input_file',
- filename: file.filename,
- file_data: `data:application/pdf;base64,${content}`,
- });
- } else if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {
- result.documents.push({
- type: 'media',
- mimeType: 'application/pdf',
- data: content,
- });
- } else if (isOpenAILikeProvider(provider) && provider != Providers.AZURE) {
- result.documents.push({
- type: 'file',
- file: {
- filename: file.filename,
- file_data: `data:application/pdf;base64,${content}`,
- },
- });
+ const block = formatDocumentBlock(
+ provider,
+ mimeType,
+ content,
+ file.filename,
+ useResponsesApi,
+ );
+ if (block) {
+ result.documents.push(block);
+ result.files.push(metadata);
+ }
+ } else if (isDocSupported && !isBedrock) {
+ const paddingChars = content.endsWith('==') ? 2 : content.endsWith('=') ? 1 : 0;
+ const decodedByteCount = Math.floor((content.length * 3) / 4) - paddingChars;
+ if (configuredFileSizeLimit && decodedByteCount > configuredFileSizeLimit) {
+ throw new Error(
+ `File size (~${(decodedByteCount / 1024 / 1024).toFixed(1)}MB) exceeds the configured limit for ${provider}`,
+ );
+ }
+
+ const block = formatDocumentBlock(
+ provider,
+ mimeType,
+ content,
+ file.filename,
+ useResponsesApi,
+ );
+ if (block) {
+ result.documents.push(block);
+ result.files.push(metadata);
}
- result.files.push(metadata);
}
}
diff --git a/packages/api/src/files/encode/processAttachments.spec.ts b/packages/api/src/files/encode/processAttachments.spec.ts
new file mode 100644
index 0000000000..91d6de4a0c
--- /dev/null
+++ b/packages/api/src/files/encode/processAttachments.spec.ts
@@ -0,0 +1,158 @@
+import {
+ FileSources,
+ mergeFileConfig,
+ EModelEndpoint,
+ getEndpointFileConfig,
+ isBedrockDocumentType,
+} from 'librechat-data-provider';
+import type { FileConfig, EndpointFileConfig } from 'librechat-data-provider';
+
+/**
+ * Mirrors the categorization logic from BaseClient.processAttachments.
+ * Extracted here for testability since the /api workspace test setup is broken.
+ */
+function categorizeFile(
+ file: {
+ type?: string | null;
+ source?: string;
+ embedded?: boolean;
+ metadata?: { fileIdentifier?: string };
+ },
+ isBedrock: boolean,
+ mergedFileConfig: FileConfig | undefined,
+ endpointFileConfig: EndpointFileConfig | undefined,
+): 'images' | 'documents' | 'videos' | 'audios' | 'skipped' {
+ const source = file.source ?? FileSources.local;
+ if (source === FileSources.text) {
+ return 'skipped';
+ }
+ if (file.embedded === true || file.metadata?.fileIdentifier != null) {
+ return 'skipped';
+ }
+
+ if (file.type?.startsWith('image/')) {
+ return 'images';
+ } else if (file.type === 'application/pdf') {
+ return 'documents';
+ } else if (isBedrock && file.type && isBedrockDocumentType(file.type)) {
+ return 'documents';
+ } else if (file.type?.startsWith('video/')) {
+ return 'videos';
+ } else if (file.type?.startsWith('audio/')) {
+ return 'audios';
+ } else if (
+ file.type &&
+ mergedFileConfig &&
+ endpointFileConfig?.supportedMimeTypes &&
+ mergedFileConfig.checkType?.(file.type, endpointFileConfig.supportedMimeTypes)
+ ) {
+ return 'documents';
+ }
+
+ return 'skipped';
+}
+
+describe('processAttachments — supportedMimeTypes routing logic', () => {
+ const endpoint = EModelEndpoint.openAI;
+
+ function resolveConfig(mimePatterns: string[]) {
+ const merged = mergeFileConfig({
+ endpoints: {
+ [endpoint]: { supportedMimeTypes: mimePatterns },
+ },
+ });
+ const epConfig = getEndpointFileConfig({
+ fileConfig: merged,
+ endpoint,
+ });
+ return { merged, epConfig };
+ }
+
+ it('should route text/csv to documents when supportedMimeTypes includes it', () => {
+ const { merged, epConfig } = resolveConfig(['text/csv']);
+ const result = categorizeFile({ type: 'text/csv' }, false, merged, epConfig);
+ expect(result).toBe('documents');
+ });
+
+ it('should route text/plain to documents when supportedMimeTypes uses wildcard', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ const result = categorizeFile({ type: 'text/plain' }, false, merged, epConfig);
+ expect(result).toBe('documents');
+ });
+
+ it('should skip application/zip when supportedMimeTypes only allows text types', () => {
+ const { merged, epConfig } = resolveConfig(['text/csv', 'text/plain']);
+ const result = categorizeFile({ type: 'application/zip' }, false, merged, epConfig);
+ expect(result).toBe('skipped');
+ });
+
+ it('should skip files when no fileConfig is provided', () => {
+ const result = categorizeFile({ type: 'text/csv' }, false, undefined, undefined);
+ expect(result).toBe('skipped');
+ });
+
+ it('should skip files with null type even with permissive config', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ const result = categorizeFile({ type: null }, false, merged, epConfig);
+ expect(result).toBe('skipped');
+ });
+
+ it('should skip files with undefined type even with permissive config', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ const result = categorizeFile({ type: undefined }, false, merged, epConfig);
+ expect(result).toBe('skipped');
+ });
+
+ it('should still route image types through images category (not documents)', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'image/png' }, false, merged, epConfig)).toBe('images');
+ });
+
+ it('should still route PDF through documents (dedicated branch)', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'application/pdf' }, false, merged, epConfig)).toBe('documents');
+ });
+
+ it('should still route video types through videos category', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'video/mp4' }, false, merged, epConfig)).toBe('videos');
+ });
+
+ it('should still route audio types through audios category', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'audio/mp3' }, false, merged, epConfig)).toBe('audios');
+ });
+
+ it('should route Bedrock document types through documents for Bedrock provider', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'text/csv' }, true, merged, epConfig)).toBe('documents');
+ });
+
+ it('should route non-Bedrock-document types for Bedrock when config allows them', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ expect(categorizeFile({ type: 'application/zip' }, true, merged, epConfig)).toBe('documents');
+ });
+
+ it('should route xlsx to documents with matching config', () => {
+ const xlsxType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
+ const { merged, epConfig } = resolveConfig([xlsxType]);
+ expect(categorizeFile({ type: xlsxType }, false, merged, epConfig)).toBe('documents');
+ });
+
+ it('should skip text source files regardless of config', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ const result = categorizeFile(
+ { type: 'text/csv', source: FileSources.text },
+ false,
+ merged,
+ epConfig,
+ );
+ expect(result).toBe('skipped');
+ });
+
+ it('should skip embedded files regardless of config', () => {
+ const { merged, epConfig } = resolveConfig(['.*']);
+ const result = categorizeFile({ type: 'text/csv', embedded: true }, false, merged, epConfig);
+ expect(result).toBe('skipped');
+ });
+});