Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
📎 feat: Direct Provider Attachment Support for Multimodal Content (#9994)
Danny Avilabcd97aad2ff9
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.jsindex 2458dc0ab3..32c76523f7 100644--- a/api/app/clients/BaseClient.js+++ b/api/app/clients/BaseClient.js@@ -1,18 +1,24 @@const crypto = require('crypto');const fetch = require('node-fetch');const { logger } = require('@librechat/data-schemas');-const { getBalanceConfig } = require('@librechat/api');const {- supportsBalanceCheck,- isAgentsEndpoint,- isParamEndpoint,- EModelEndpoint,+ getBalanceConfig,+ encodeAndFormatAudios,+ encodeAndFormatVideos,+ encodeAndFormatDocuments,+} = require('@librechat/api');+const {+ Constants,+ ErrorTypes,ContentTypes,excludedKeys,- ErrorTypes,- Constants,+ EModelEndpoint,+ isParamEndpoint,+ isAgentsEndpoint,+ supportsBalanceCheck,} = require('librechat-data-provider');const { getMessages, saveMessage, updateMessage, saveConvo, getConvo } = require('~/models');+const { getStrategyFunctions } = require('~/server/services/Files/strategies');const { checkBalance } = require('~/models/balanceMethods');const { truncateToolCallOutputs } = require('./prompts');const { getFiles } = require('~/models/File');@@ -1198,8 +1204,99 @@ class BaseClient {return await this.sendCompletion(payload, opts);}+ async addDocuments(message, attachments) {+ const documentResult = await encodeAndFormatDocuments(+ this.options.req,+ attachments,+ {+ provider: this.options.agent?.provider,+ useResponsesApi: this.options.agent?.model_parameters?.useResponsesApi,+ },+ getStrategyFunctions,+ );+ message.documents =+ documentResult.documents && documentResult.documents.length+ ? documentResult.documents+ : undefined;+ return documentResult.files;+ }++ async addVideos(message, attachments) {+ const videoResult = await encodeAndFormatVideos(+ this.options.req,+ attachments,+ this.options.agent.provider,+ getStrategyFunctions,+ );+ message.videos =+ videoResult.videos && videoResult.videos.length ? videoResult.videos : undefined;+ return videoResult.files;+ }++ async addAudios(message, attachments) {+ const audioResult = await encodeAndFormatAudios(+ this.options.req,+ attachments,+ this.options.agent.provider,+ getStrategyFunctions,+ );+ message.audios =+ audioResult.audios && audioResult.audios.length ? audioResult.audios : undefined;+ return audioResult.files;+ }++ async processAttachments(message, attachments) {+ const categorizedAttachments = {+ images: [],+ documents: [],+ videos: [],+ audios: [],+ };++ for (const file of attachments) {+ if (file.type.startsWith('image/')) {+ categorizedAttachments.images.push(file);+ } else if (file.type === 'application/pdf') {+ categorizedAttachments.documents.push(file);+ } else if (file.type.startsWith('video/')) {+ categorizedAttachments.videos.push(file);+ } else if (file.type.startsWith('audio/')) {+ categorizedAttachments.audios.push(file);+ }+ }++ const [imageFiles, documentFiles, videoFiles, audioFiles] = await Promise.all([+ categorizedAttachments.images.length > 0+ ? this.addImageURLs(message, categorizedAttachments.images)+ : Promise.resolve([]),+ categorizedAttachments.documents.length > 0+ ? this.addDocuments(message, categorizedAttachments.documents)+ : Promise.resolve([]),+ categorizedAttachments.videos.length > 0+ ? this.addVideos(message, categorizedAttachments.videos)+ : Promise.resolve([]),+ categorizedAttachments.audios.length > 0+ ? this.addAudios(message, categorizedAttachments.audios)+ : Promise.resolve([]),+ ]);++ const allFiles = [...imageFiles, ...documentFiles, ...videoFiles, ...audioFiles];+ const seenFileIds = new Set();+ const uniqueFiles = [];++ for (const file of allFiles) {+ if (file.file_id && !seenFileIds.has(file.file_id)) {+ seenFileIds.add(file.file_id);+ uniqueFiles.push(file);+ } else if (!file.file_id) {+ uniqueFiles.push(file);+ }+ }++ return uniqueFiles;+ }+/**- ** @param {TMessage[]} _messages* @returns {Promise<TMessage[]>}*/@@ -1248,7 +1345,7 @@ class BaseClient {{},);- await this.addImageURLs(message, files, this.visionMode);+ await this.processAttachments(message, files);this.message_file_map[message.messageId] = files;return message;diff --git a/api/package.json b/api/package.jsonindex 01c931a652..73cb0633e9 100644--- a/api/package.json+++ b/api/package.json@@ -48,7 +48,7 @@"@langchain/google-genai": "^0.2.13","@langchain/google-vertexai": "^0.2.13","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.4.84",+ "@librechat/agents": "^2.4.85","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex bf32385162..a9f5543a61 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -257,7 +257,7 @@ class AgentClient extends BaseClient {};}- const files = await this.addImageURLs(+ const files = await this.processAttachments(orderedMessages[orderedMessages.length - 1],attachments,);diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.jsindex 16e75ba200..d3a3a21538 100644--- a/api/server/services/Files/Local/crud.js+++ b/api/server/services/Files/Local/crud.js@@ -4,6 +4,7 @@ const axios = require('axios');const { logger } = require('@librechat/data-schemas');const { EModelEndpoint } = require('librechat-data-provider');const { generateShortLivedToken } = require('@librechat/api');+const { resizeImageBuffer } = require('~/server/services/Files/images/resize');const { getBufferMetadata } = require('~/server/utils');const paths = require('~/config/paths');@@ -286,7 +287,18 @@ async function uploadLocalFile({ req, file, file_id }) {await fs.promises.writeFile(newPath, inputBuffer);const filepath = path.posix.join('/', 'uploads', req.user.id, path.basename(newPath));- return { filepath, bytes };+ let height, width;+ if (file.mimetype && file.mimetype.startsWith('image/')) {+ try {+ const { width: imgWidth, height: imgHeight } = await resizeImageBuffer(inputBuffer, 'high');+ height = imgHeight;+ width = imgWidth;+ } catch (error) {+ logger.warn('[uploadLocalFile] Could not get image dimensions:', error.message);+ }+ }++ return { filepath, bytes, height, width };}/**diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.jsindex c8221a6de5..f7220715f6 100644--- a/api/server/services/Files/process.js+++ b/api/server/services/Files/process.js@@ -522,11 +522,6 @@ const processAgentFileUpload = async ({ req, res, metadata }) => {}const isImage = file.mimetype.startsWith('image');- if (!isImage && !tool_resource) {- /** Note: this needs to be removed when we can support files to providers */- throw new Error('No tool resource provided for non-image agent file upload');- }-let fileInfoMetadata;const entity_id = messageAttachment === true ? undefined : agent_id;const basePath = mime.getType(file.originalname)?.startsWith('image') ? 'images' : 'uploads';diff --git a/client/src/Providers/DragDropContext.tsx b/client/src/Providers/DragDropContext.tsxindex a86af6510b..3a5fe2924c 100644--- a/client/src/Providers/DragDropContext.tsx+++ b/client/src/Providers/DragDropContext.tsx@@ -1,23 +1,38 @@import React, { createContext, useContext, useMemo } from 'react';+import type { EModelEndpoint } from 'librechat-data-provider';+import { useGetEndpointsQuery } from '~/data-provider';+import { getEndpointField } from '~/utils/endpoints';import { useChatContext } from './ChatContext';interface DragDropContextValue {conversationId: string | null | undefined;agentId: string | null | undefined;+ endpoint: string | null | undefined;+ endpointType?: EModelEndpoint | undefined;}const DragDropContext = createContext<DragDropContextValue | undefined>(undefined);export function DragDropProvider({ children }: { children: React.ReactNode }) {const { conversation } = useChatContext();+ const { data: endpointsConfig } = useGetEndpointsQuery();++ const endpointType = useMemo(() => {+ return (+ getEndpointField(endpointsConfig, conversation?.endpoint, 'type') ||+ (conversation?.endpoint as EModelEndpoint | undefined)+ );+ }, [conversation?.endpoint, endpointsConfig]);/** Context value only created when conversation fields change */const contextValue = useMemo<DragDropContextValue>(() => ({conversationId: conversation?.conversationId,agentId: conversation?.agent_id,+ endpoint: conversation?.endpoint,+ endpointType: endpointType,}),- [conversation?.conversationId, conversation?.agent_id],+ [conversation?.conversationId, conversation?.agent_id, conversation?.endpoint, endpointType],);return <DragDropContext.Provider value={contextValue}>{children}</DragDropContext.Provider>;diff --git a/client/src/components/Chat/Input/Files/AttachFileChat.tsx b/client/src/components/Chat/Input/Files/AttachFileChat.tsxindex f80b8aef82..4757c598a5 100644--- a/client/src/components/Chat/Input/Files/AttachFileChat.tsx+++ b/client/src/components/Chat/Input/Files/AttachFileChat.tsx@@ -2,13 +2,15 @@ import { memo, useMemo } from 'react';import {Constants,supportsFiles,+ EModelEndpoint,mergeFileConfig,isAgentsEndpoint,isAssistantsEndpoint,fileConfig as defaultFileConfig,} from 'librechat-data-provider';import type { EndpointFileConfig, TConversation } from 'librechat-data-provider';-import { useGetFileConfig } from '~/data-provider';+import { useGetFileConfig, useGetEndpointsQuery } from '~/data-provider';+import { getEndpointField } from '~/utils/endpoints';import AttachFileMenu from './AttachFileMenu';import AttachFile from './AttachFile';@@ -20,7 +22,7 @@ function AttachFileChat({conversation: TConversation | null;}) {const conversationId = conversation?.conversationId ?? Constants.NEW_CONVO;- const { endpoint, endpointType } = conversation ?? { endpoint: null };+ const { endpoint } = conversation ?? { endpoint: null };const isAgents = useMemo(() => isAgentsEndpoint(endpoint), [endpoint]);const isAssistants = useMemo(() => isAssistantsEndpoint(endpoint), [endpoint]);@@ -28,6 +30,15 @@ function AttachFileChat({select: (data) => mergeFileConfig(data),});+ const { data: endpointsConfig } = useGetEndpointsQuery();++ const endpointType = useMemo(() => {+ return (+ getEndpointField(endpointsConfig, endpoint, 'type') ||+ (endpoint as EModelEndpoint | undefined)+ );+ }, [endpoint, endpointsConfig]);+const endpointFileConfig = fileConfig.endpoints[endpoint ?? ''] as EndpointFileConfig | undefined;const endpointSupportsFiles: boolean = supportsFiles[endpointType ?? endpoint ?? ''] ?? false;const isUploadDisabled = (disableInputs || endpointFileConfig?.disabled) ?? false;@@ -37,7 +48,9 @@ function AttachFileChat({} else if (isAgents || (endpointSupportsFiles && !isUploadDisabled)) {return (<AttachFileMenu+ endpoint={endpoint}disabled={disableInputs}+ endpointType={endpointType}conversationId={conversationId}agentId={conversation?.agent_id}endpointFileConfig={endpointFileConfig}diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsxindex 75398c2633..fa56cdf08b 100644--- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx+++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx@@ -1,8 +1,19 @@import React, { useRef, useState, useMemo } from 'react';-import * as Ariakit from '@ariakit/react';import { useRecoilState } from 'recoil';-import { FileSearch, ImageUpIcon, TerminalSquareIcon, FileType2Icon } from 'lucide-react';-import { EToolResources, EModelEndpoint, defaultAgentCapabilities } from 'librechat-data-provider';+import * as Ariakit from '@ariakit/react';+import {+ FileSearch,+ ImageUpIcon,+ FileType2Icon,+ FileImageIcon,+ TerminalSquareIcon,+} from 'lucide-react';+import {+ EToolResources,+ EModelEndpoint,+ defaultAgentCapabilities,+ isDocumentSupportedProvider,+} from 'librechat-data-provider';import {FileUpload,TooltipAnchor,@@ -26,15 +37,19 @@ import { MenuItemProps } from '~/common';import { cn } from '~/utils';interface AttachFileMenuProps {- conversationId: string;agentId?: string | null;+ endpoint?: string | null;disabled?: boolean | null;+ conversationId: string;+ endpointType?: EModelEndpoint;endpointFileConfig?: EndpointFileConfig;}const AttachFileMenu = ({agentId,+ endpoint,disabled,+ endpointType,conversationId,endpointFileConfig,}: AttachFileMenuProps) => {@@ -55,44 +70,75 @@ const AttachFileMenu = ({overrideEndpointFileConfig: endpointFileConfig,toolResource,});++ const { agentsConfig } = useGetAgentsConfig();const { data: startupConfig } = useGetStartupConfig();const sharePointEnabled = startupConfig?.sharePointFilePickerEnabled;const [isSharePointDialogOpen, setIsSharePointDialogOpen] = useState(false);- const { agentsConfig } = useGetAgentsConfig();+/** TODO: Ephemeral Agent Capabilities* Allow defining agent capabilities on a per-endpoint basis* Use definition for agents endpoint for ephemeral agents* */const capabilities = useAgentCapabilities(agentsConfig?.capabilities ?? defaultAgentCapabilities);- const { fileSearchAllowedByAgent, codeAllowedByAgent } = useAgentToolPermissions(+ const { fileSearchAllowedByAgent, codeAllowedByAgent, provider } = useAgentToolPermissions(agentId,ephemeralAgent,);- const handleUploadClick = (isImage?: boolean) => {+ const handleUploadClick = (+ fileType?: 'image' | 'document' | 'multimodal' | 'google_multimodal',+ ) => {if (!inputRef.current) {return;}inputRef.current.value = '';- inputRef.current.accept = isImage === true ? 'image/*' : '';+ if (fileType === 'image') {+ inputRef.current.accept = 'image/*';+ } else if (fileType === 'document') {+ inputRef.current.accept = '.pdf,application/pdf';+ } else if (fileType === 'multimodal') {+ inputRef.current.accept = 'image/*,.pdf,application/pdf';+ } else if (fileType === 'google_multimodal') {+ inputRef.current.accept = 'image/*,.pdf,application/pdf,video/*,audio/*';+ } else {+ inputRef.current.accept = '';+ }inputRef.current.click();inputRef.current.accept = '';};const dropdownItems = useMemo(() => {- const createMenuItems = (onAction: (isImage?: boolean) => void) => {- const items: MenuItemProps[] = [- {+ const createMenuItems = (+ onAction: (fileType?: 'image' | 'document' | 'multimodal' | 'google_multimodal') => void,+ ) => {+ const items: MenuItemProps[] = [];++ const currentProvider = provider || endpoint;++ if (isDocumentSupportedProvider(endpointType || currentProvider)) {+ items.push({+ label: localize('com_ui_upload_provider'),+ onClick: () => {+ setToolResource(undefined);+ onAction(+ (provider || endpoint) === EModelEndpoint.google ? 'google_multimodal' : 'multimodal',+ );+ },+ icon: <FileImageIcon className="icon-md" />,+ });+ } else {+ items.push({label: localize('com_ui_upload_image_input'),onClick: () => {setToolResource(undefined);- onAction(true);+ onAction('image');},icon: <ImageUpIcon className="icon-md" />,- },- ];+ });+ }if (capabilities.contextEnabled) {items.push({@@ -156,8 +202,11 @@ const AttachFileMenu = ({return localItems;}, [- capabilities,localize,+ endpoint,+ provider,+ endpointType,+ capabilities,setToolResource,setEphemeralAgent,sharePointEnabled,diff --git a/client/src/components/Chat/Input/Files/DragDropModal.tsx b/client/src/components/Chat/Input/Files/DragDropModal.tsxindex e9992c4dcb..209972e4a8 100644--- a/client/src/components/Chat/Input/Files/DragDropModal.tsx+++ b/client/src/components/Chat/Input/Files/DragDropModal.tsx@@ -1,8 +1,18 @@import React, { useMemo } from 'react';import { useRecoilValue } from 'recoil';import { OGDialog, OGDialogTemplate } from '@librechat/client';-import { EToolResources, defaultAgentCapabilities } from 'librechat-data-provider';-import { ImageUpIcon, FileSearch, TerminalSquareIcon, FileType2Icon } from 'lucide-react';+import {+ EToolResources,+ defaultAgentCapabilities,+ isDocumentSupportedProvider,+} from 'librechat-data-provider';+import {+ ImageUpIcon,+ FileSearch,+ FileType2Icon,+ FileImageIcon,+ TerminalSquareIcon,+} from 'lucide-react';import {useAgentToolPermissions,useAgentCapabilities,@@ -34,22 +44,34 @@ const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragD* Use definition for agents endpoint for ephemeral agents* */const capabilities = useAgentCapabilities(agentsConfig?.capabilities ?? defaultAgentCapabilities);- const { conversationId, agentId } = useDragDropContext();+ const { conversationId, agentId, endpoint, endpointType } = useDragDropContext();const ephemeralAgent = useRecoilValue(ephemeralAgentByConvoId(conversationId ?? ''));- const { fileSearchAllowedByAgent, codeAllowedByAgent } = useAgentToolPermissions(+ const { fileSearchAllowedByAgent, codeAllowedByAgent, provider } = useAgentToolPermissions(agentId,ephemeralAgent,);const options = useMemo(() => {- const _options: FileOption[] = [- {+ const _options: FileOption[] = [];+ const currentProvider = provider || endpoint;++ // Check if provider supports document upload+ if (isDocumentSupportedProvider(endpointType || currentProvider)) {+ _options.push({+ label: localize('com_ui_upload_provider'),+ value: undefined,+ icon: <FileImageIcon className="icon-md" />,+ condition: true, // Allow for both images and documents+ });+ } else {+ // Only show image upload option if all files are images and provider doesn't support documents+ _options.push({label: localize('com_ui_upload_image_input'),value: undefined,icon: <ImageUpIcon className="icon-md" />,condition: files.every((file) => file.type?.startsWith('image/')),- },- ];+ });+ }if (capabilities.fileSearchEnabled && fileSearchAllowedByAgent) {_options.push({label: localize('com_ui_upload_file_search'),@@ -73,7 +95,16 @@ const DragDropModal = ({ onOptionSelect, setShowModal, files, isVisible }: DragD}return _options;- }, [capabilities, files, localize, fileSearchAllowedByAgent, codeAllowedByAgent]);+ }, [+ files,+ localize,+ provider,+ endpoint,+ endpointType,+ capabilities,+ codeAllowedByAgent,+ fileSearchAllowedByAgent,+ ]);if (!isVisible) {return null;diff --git a/client/src/components/Chat/Messages/Content/Part.tsx b/client/src/components/Chat/Messages/Content/Part.tsxindex 5d98b5aca0..aa9f4da82d 100644--- a/client/src/components/Chat/Messages/Content/Part.tsx+++ b/client/src/components/Chat/Messages/Content/Part.tsx@@ -57,7 +57,7 @@ const Part = memo(</>);} else if (part.type === ContentTypes.TEXT) {- const text = typeof part.text === 'string' ? part.text : part.text.value;+ const text = typeof part.text === 'string' ? part.text : part.text?.value;if (typeof text !== 'string') {return null;@@ -71,7 +71,7 @@ const Part = memo(</Container>);} else if (part.type === ContentTypes.THINK) {- const reasoning = typeof part.think === 'string' ? part.think : part.think.value;+ const reasoning = typeof part.think === 'string' ? part.think : part.think?.value;if (typeof reasoning !== 'string') {return null;}diff --git a/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx b/client/src/components/Chat/Messages/Content/Parts/LogContent.tsxindex 0d53fb50ee..d2a303f49f 100644--- a/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx+++ b/client/src/components/Chat/Messages/Content/Parts/LogContent.tsx@@ -37,7 +37,7 @@ const LogContent: React.FC<LogContentProps> = ({ output = '', renderImages, attaattachments?.forEach((attachment) => {const { width, height, filepath = null } = attachment as TFile & TAttachmentMetadata;const isImage =- imageExtRegex.test(attachment.filename) &&+ imageExtRegex.test(attachment.filename ?? '') &&width != null &&height != null &&filepath != null;@@ -56,21 +56,25 @@ const LogContent: React.FC<LogContentProps> = ({ output = '', renderImages, attaconst renderAttachment = (file: TAttachment) => {const now = new Date();- const expiresAt = typeof file.expiresAt === 'number' ? new Date(file.expiresAt) : null;+ const expiresAt =+ 'expiresAt' in file && typeof file.expiresAt === 'number' ? new Date(file.expiresAt) : null;const isExpired = expiresAt ? isAfter(now, expiresAt) : false;+ const filename = file.filename || '';if (isExpired) {- return `${file.filename} ${localize('com_download_expired')}`;+ return `${filename} ${localize('com_download_expired')}`;}+ const filepath = file.filepath || '';+// const expirationText = expiresAt// ? ` ${localize('com_download_expires', { 0: format(expiresAt, 'MM/dd/yy HH:mm') })}`// : ` ${localize('com_click_to_download')}`;return (- <LogLink href={file.filepath} filename={file.filename}>+ <LogLink href={filepath} filename={filename}>{'- '}- {file.filename} {localize('com_click_to_download')}+ {filename} {localize('com_click_to_download')}</LogLink>);};diff --git a/client/src/hooks/Agents/useAgentToolPermissions.ts b/client/src/hooks/Agents/useAgentToolPermissions.tsindex 90f2bc88e5..eea549d7a6 100644--- a/client/src/hooks/Agents/useAgentToolPermissions.ts+++ b/client/src/hooks/Agents/useAgentToolPermissions.ts@@ -9,6 +9,7 @@ interface AgentToolPermissionsResult {fileSearchAllowedByAgent: boolean;codeAllowedByAgent: boolean;tools: string[] | undefined;+ provider?: string;}/**@@ -36,6 +37,8 @@ export default function useAgentToolPermissions([agentData?.tools, selectedAgent?.tools],);+ const provider = useMemo(() => selectedAgent?.provider, [selectedAgent?.provider]);+const fileSearchAllowedByAgent = useMemo(() => {// Check ephemeral agent settingsif (isEphemeralAgent(agentId)) {@@ -61,6 +64,7 @@ export default function useAgentToolPermissions(return {fileSearchAllowedByAgent,codeAllowedByAgent,+ provider,tools,};}diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.tsindex 9deaa78f24..7825888985 100644--- a/client/src/hooks/Files/useFileHandling.ts+++ b/client/src/hooks/Files/useFileHandling.ts@@ -392,13 +392,6 @@ const useFileHandling = (params?: UseFileHandling) => {} else {// File wasn't processed, proceed with originalconst isImage = originalFile.type.split('/')[0] === 'image';- const tool_resource =- initialExtendedFile.tool_resource ?? params?.additionalMetadata?.tool_resource;- if (isAgentsEndpoint(endpoint) && !isImage && tool_resource == null) {- /** Note: this needs to be removed when we can support files to providers */- setError('com_error_files_unsupported_capability');- continue;- }// Update progress to show ready for uploadconst readyExtendedFile = {diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.jsonindex 0facdb493a..c4b8481398 100644--- a/client/src/locales/en/translation.json+++ b/client/src/locales/en/translation.json@@ -363,7 +363,6 @@"com_error_files_dupe": "Duplicate file detected.","com_error_files_empty": "Empty files are not allowed.","com_error_files_process": "An error occurred while processing the file.",- "com_error_files_unsupported_capability": "No capabilities enabled that support this file type.","com_error_files_upload": "An error occurred while uploading the file.","com_error_files_upload_canceled": "The file upload request was canceled. Note: the file upload may still be processing and will need to be manually deleted.","com_error_files_validation": "An error occurred while validating the file.",@@ -1230,6 +1229,7 @@"com_ui_upload_invalid": "Invalid file for upload. Must be an image not exceeding the limit","com_ui_upload_invalid_var": "Invalid file for upload. Must be an image not exceeding {{0}} MB","com_ui_upload_ocr_text": "Upload as Text",+ "com_ui_upload_provider": "Upload to Provider","com_ui_upload_success": "Successfully uploaded file","com_ui_upload_type": "Select Upload Type","com_ui_usage": "Usage",diff --git a/client/src/utils/files.ts b/client/src/utils/files.tsindex 81830c64ff..0959ba91bb 100644--- a/client/src/utils/files.ts+++ b/client/src/utils/files.ts@@ -1,4 +1,11 @@-import { SheetPaths, TextPaths, FilePaths, CodePaths } from '@librechat/client';+import {+ TextPaths,+ FilePaths,+ CodePaths,+ AudioPaths,+ VideoPaths,+ SheetPaths,+} from '@librechat/client';import {megabyte,QueryKeys,@@ -38,6 +45,18 @@ const artifact = {title: 'Code',};+const audioFile = {+ paths: AudioPaths,+ fill: '#FF6B35',+ title: 'Audio',+};++const videoFile = {+ paths: VideoPaths,+ fill: '#8B5CF6',+ title: 'Video',+};+export const fileTypes = {/* Category matches */file: {@@ -47,6 +66,8 @@ export const fileTypes = {},text: textDocument,txt: textDocument,+ audio: audioFile,+ video: videoFile,// application:,/* Partial matches */diff --git a/client/src/utils/messages.ts b/client/src/utils/messages.tsindex 7a52ff4106..caae46d923 100644--- a/client/src/utils/messages.ts+++ b/client/src/utils/messages.ts@@ -25,7 +25,7 @@ export const getLatestText = (message?: TMessage | null, includeIndex?: boolean)continue;}- const text = (typeof part?.text === 'string' ? part.text : part?.text.value) ?? '';+ const text = (typeof part?.text === 'string' ? part.text : part?.text?.value) ?? '';if (text.length > 0) {if (includeIndex === true) {return `${text}-${i}`;@@ -52,7 +52,12 @@ export const getAllContentText = (message?: TMessage | null): string => {if (message.content && message.content.length > 0) {return message.content.filter((part) => part.type === ContentTypes.TEXT)- .map((part) => (typeof part.text === 'string' ? part.text : part.text.value) || '')+ .map((part) => {+ if (!('text' in part)) return '';+ const text = part.text;+ if (typeof text === 'string') return text;+ return text?.value || '';+ }).filter((text) => text.length > 0).join('\n');}diff --git a/package-lock.json b/package-lock.jsonindex 019317a2ea..72bc3a7af6 100644--- a/package-lock.json+++ b/package-lock.json@@ -64,7 +64,7 @@"@langchain/google-genai": "^0.2.13","@langchain/google-vertexai": "^0.2.13","@langchain/textsplitters": "^0.1.0",- "@librechat/agents": "^2.4.84",+ "@librechat/agents": "^2.4.85","@librechat/api": "*","@librechat/data-schemas": "*","@microsoft/microsoft-graph-client": "^3.0.7",@@ -21522,9 +21522,9 @@}},"node_modules/@librechat/agents": {- "version": "2.4.84",- "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.4.84.tgz",- "integrity": "sha512-wOPqv5yQfhkuBZ29FrJGUdDMCIvcnqUAigFeoPU8QOeqi+S9rRobx2+2D3+JbbgSsDL5yO7SyxGEHkQ7A6xZDQ==",+ "version": "2.4.85",+ "resolved": "https://registry.npmjs.org/@librechat/agents/-/agents-2.4.85.tgz",+ "integrity": "sha512-t6h5f6ApnoEC+x8kqBlke1RR6BPzT+9BvlkA8VxvQVJtYIt5Ey4BOTRDGjdilDoXUcLui11PbjCd17EbjPkTcA==","license": "MIT","dependencies": {"@langchain/anthropic": "^0.3.26",@@ -51336,7 +51336,7 @@"@azure/storage-blob": "^12.27.0","@keyv/redis": "^4.3.3","@langchain/core": "^0.3.62",- "@librechat/agents": "^2.4.84",+ "@librechat/agents": "^2.4.85","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.17.1","axios": "^1.12.1",diff --git a/packages/api/package.json b/packages/api/package.jsonindex 3ea7d343e4..05054498e2 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -80,7 +80,7 @@"@azure/storage-blob": "^12.27.0","@keyv/redis": "^4.3.3","@langchain/core": "^0.3.62",- "@librechat/agents": "^2.4.84",+ "@librechat/agents": "^2.4.85","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.17.1","axios": "^1.12.1",diff --git a/packages/api/src/files/encode/audio.ts b/packages/api/src/files/encode/audio.tsnew file mode 100644index 0000000000..d411e61640--- /dev/null+++ b/packages/api/src/files/encode/audio.ts@@ -0,0 +1,74 @@+import { Providers } from '@librechat/agents';+import { isDocumentSupportedProvider } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';+import type { Request } from 'express';+import type { StrategyFunctions, AudioResult } from '~/types/files';+import { validateAudio } from '~/files/validation';+import { getFileStream } from './utils';++/**+ * Encodes and formats audio files for different providers+ * @param req - The request object+ * @param files - Array of audio files+ * @param provider - The provider to format for (currently only google is supported)+ * @param getStrategyFunctions - Function to get strategy functions+ * @returns Promise that resolves to audio and file metadata+ */+export async function encodeAndFormatAudios(+ req: Request,+ files: IMongoFile[],+ provider: Providers,+ getStrategyFunctions: (source: string) => StrategyFunctions,+): Promise<AudioResult> {+ if (!files?.length) {+ return { audios: [], files: [] };+ }++ const encodingMethods: Record<string, StrategyFunctions> = {};+ const result: AudioResult = { audios: [], files: [] };++ const results = await Promise.allSettled(+ files.map((file) => getFileStream(req, file, encodingMethods, getStrategyFunctions)),+ );++ for (const settledResult of results) {+ if (settledResult.status === 'rejected') {+ console.error('Audio processing failed:', settledResult.reason);+ continue;+ }++ const processed = settledResult.value;+ if (!processed) continue;++ const { file, content, metadata } = processed;++ if (!content || !file) {+ if (metadata) result.files.push(metadata);+ continue;+ }++ if (!file.type.startsWith('audio/') || !isDocumentSupportedProvider(provider)) {+ result.files.push(metadata);+ continue;+ }++ const audioBuffer = Buffer.from(content, 'base64');+ const validation = await validateAudio(audioBuffer, audioBuffer.length, provider);++ if (!validation.isValid) {+ throw new Error(`Audio validation failed: ${validation.error}`);+ }++ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ result.audios.push({+ type: 'audio',+ mimeType: file.type,+ data: content,+ });+ }++ result.files.push(metadata);+ }++ return result;+}diff --git a/packages/api/src/files/encode/document.ts b/packages/api/src/files/encode/document.tsnew file mode 100644index 0000000000..bc1396958c--- /dev/null+++ b/packages/api/src/files/encode/document.ts@@ -0,0 +1,108 @@+import { Providers } from '@librechat/agents';+import { isOpenAILikeProvider, isDocumentSupportedProvider } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';+import type { Request } from 'express';+import type { StrategyFunctions, DocumentResult } from '~/types/files';+import { validatePdf } from '~/files/validation';+import { getFileStream } from './utils';++/**+ * Processes and encodes document files for various providers+ * @param req - Express request object+ * @param files - Array of file objects to process+ * @param provider - The provider name+ * @param getStrategyFunctions - Function to get strategy functions+ * @returns Promise that resolves to documents and file metadata+ */+export async function encodeAndFormatDocuments(+ req: Request,+ files: IMongoFile[],+ { provider, useResponsesApi }: { provider: Providers; useResponsesApi?: boolean },+ getStrategyFunctions: (source: string) => StrategyFunctions,+): Promise<DocumentResult> {+ if (!files?.length) {+ return { documents: [], files: [] };+ }++ const encodingMethods: Record<string, StrategyFunctions> = {};+ const result: DocumentResult = { documents: [], files: [] };++ const documentFiles = files.filter(+ (file) => file.type === 'application/pdf' || file.type?.startsWith('application/'),+ );++ if (!documentFiles.length) {+ return result;+ }++ const results = await Promise.allSettled(+ documentFiles.map((file) => {+ if (file.type !== 'application/pdf' || !isDocumentSupportedProvider(provider)) {+ return Promise.resolve(null);+ }+ return getFileStream(req, file, encodingMethods, getStrategyFunctions);+ }),+ );++ for (const settledResult of results) {+ if (settledResult.status === 'rejected') {+ console.error('Document processing failed:', settledResult.reason);+ continue;+ }++ const processed = settledResult.value;+ if (!processed) continue;++ const { file, content, metadata } = processed;++ if (!content || !file) {+ if (metadata) result.files.push(metadata);+ continue;+ }++ if (file.type === 'application/pdf' && isDocumentSupportedProvider(provider)) {+ const pdfBuffer = Buffer.from(content, 'base64');+ const validation = await validatePdf(pdfBuffer, pdfBuffer.length, provider);++ if (!validation.isValid) {+ throw new Error(`PDF validation failed: ${validation.error}`);+ }++ if (provider === Providers.ANTHROPIC) {+ result.documents.push({+ type: 'document',+ source: {+ type: 'base64',+ media_type: 'application/pdf',+ data: content,+ },+ cache_control: { type: 'ephemeral' },+ citations: { enabled: true },+ });+ } else if (useResponsesApi) {+ result.documents.push({+ type: 'input_file',+ filename: file.filename,+ file_data: `data:application/pdf;base64,${content}`,+ });+ } else if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ result.documents.push({+ type: 'document',+ mimeType: 'application/pdf',+ data: content,+ });+ } else if (isOpenAILikeProvider(provider) && provider != Providers.AZURE) {+ result.documents.push({+ type: 'file',+ file: {+ filename: file.filename,+ file_data: `data:application/pdf;base64,${content}`,+ },+ });+ }+ result.files.push(metadata);+ }+ }++ return result;+}diff --git a/packages/api/src/files/encode/index.ts b/packages/api/src/files/encode/index.tsnew file mode 100644index 0000000000..a0708596f3--- /dev/null+++ b/packages/api/src/files/encode/index.ts@@ -0,0 +1,3 @@+export * from './audio';+export * from './document';+export * from './video';diff --git a/packages/api/src/files/encode/utils.ts b/packages/api/src/files/encode/utils.tsnew file mode 100644index 0000000000..3664e074c9--- /dev/null+++ b/packages/api/src/files/encode/utils.ts@@ -0,0 +1,46 @@+import getStream from 'get-stream';+import { FileSources } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';+import type { Request } from 'express';+import type { StrategyFunctions, ProcessedFile } from '~/types/files';++/**+ * Processes a file by downloading and encoding it to base64+ * @param req - Express request object+ * @param file - File object to process+ * @param encodingMethods - Cache of encoding methods by source+ * @param getStrategyFunctions - Function to get strategy functions for a source+ * @returns Processed file with content and metadata, or null if filepath missing+ */+export async function getFileStream(+ req: Request,+ file: IMongoFile,+ encodingMethods: Record<string, StrategyFunctions>,+ getStrategyFunctions: (source: string) => StrategyFunctions,+): Promise<ProcessedFile | null> {+ if (!file?.filepath) {+ return null;+ }++ const source = file.source ?? FileSources.local;+ if (!encodingMethods[source]) {+ encodingMethods[source] = getStrategyFunctions(source);+ }++ const { getDownloadStream } = encodingMethods[source];+ const stream = await getDownloadStream(req, file.filepath);+ const buffer = await getStream.buffer(stream);++ return {+ file,+ content: buffer.toString('base64'),+ metadata: {+ file_id: file.file_id,+ temp_file_id: file.temp_file_id,+ filepath: file.filepath,+ source: file.source,+ filename: file.filename,+ type: file.type,+ },+ };+}diff --git a/packages/api/src/files/encode/video.ts b/packages/api/src/files/encode/video.tsnew file mode 100644index 0000000000..10fd4c691f--- /dev/null+++ b/packages/api/src/files/encode/video.ts@@ -0,0 +1,74 @@+import { Providers } from '@librechat/agents';+import { isDocumentSupportedProvider } from 'librechat-data-provider';+import type { IMongoFile } from '@librechat/data-schemas';+import type { Request } from 'express';+import type { StrategyFunctions, VideoResult } from '~/types/files';+import { validateVideo } from '~/files/validation';+import { getFileStream } from './utils';++/**+ * Encodes and formats video files for different providers+ * @param req - The request object+ * @param files - Array of video files+ * @param provider - The provider to format for+ * @param getStrategyFunctions - Function to get strategy functions+ * @returns Promise that resolves to videos and file metadata+ */+export async function encodeAndFormatVideos(+ req: Request,+ files: IMongoFile[],+ provider: Providers,+ getStrategyFunctions: (source: string) => StrategyFunctions,+): Promise<VideoResult> {+ if (!files?.length) {+ return { videos: [], files: [] };+ }++ const encodingMethods: Record<string, StrategyFunctions> = {};+ const result: VideoResult = { videos: [], files: [] };++ const results = await Promise.allSettled(+ files.map((file) => getFileStream(req, file, encodingMethods, getStrategyFunctions)),+ );++ for (const settledResult of results) {+ if (settledResult.status === 'rejected') {+ console.error('Video processing failed:', settledResult.reason);+ continue;+ }++ const processed = settledResult.value;+ if (!processed) continue;++ const { file, content, metadata } = processed;++ if (!content || !file) {+ if (metadata) result.files.push(metadata);+ continue;+ }++ if (!file.type.startsWith('video/') || !isDocumentSupportedProvider(provider)) {+ result.files.push(metadata);+ continue;+ }++ const videoBuffer = Buffer.from(content, 'base64');+ const validation = await validateVideo(videoBuffer, videoBuffer.length, provider);++ if (!validation.isValid) {+ throw new Error(`Video validation failed: ${validation.error}`);+ }++ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ result.videos.push({+ type: 'video',+ mimeType: file.type,+ data: content,+ });+ }++ result.files.push(metadata);+ }++ return result;+}diff --git a/packages/api/src/files/index.ts b/packages/api/src/files/index.tsindex 49e5bb4151..3d1a3118e3 100644--- a/packages/api/src/files/index.ts+++ b/packages/api/src/files/index.ts@@ -1,5 +1,7 @@export * from './audio';+export * from './encode';export * from './mistral/crud';export * from './ocr';export * from './parse';+export * from './validation';export * from './text';diff --git a/packages/api/src/files/validation.ts b/packages/api/src/files/validation.tsnew file mode 100644index 0000000000..a8394fd8f6--- /dev/null+++ b/packages/api/src/files/validation.ts@@ -0,0 +1,186 @@+import { Providers } from '@librechat/agents';+import { mbToBytes, isOpenAILikeProvider } from 'librechat-data-provider';++export interface PDFValidationResult {+ isValid: boolean;+ error?: string;+}++export interface VideoValidationResult {+ isValid: boolean;+ error?: string;+}++export interface AudioValidationResult {+ isValid: boolean;+ error?: string;+}++export async function validatePdf(+ pdfBuffer: Buffer,+ fileSize: number,+ provider: Providers,+): Promise<PDFValidationResult> {+ if (provider === Providers.ANTHROPIC) {+ return validateAnthropicPdf(pdfBuffer, fileSize);+ }++ if (isOpenAILikeProvider(provider)) {+ return validateOpenAIPdf(fileSize);+ }++ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ return validateGooglePdf(fileSize);+ }++ return { isValid: true };+}++/**+ * Validates if a PDF meets Anthropic's requirements+ * @param pdfBuffer - The PDF file as a buffer+ * @param fileSize - The file size in bytes+ * @returns Promise that resolves to validation result+ */+async function validateAnthropicPdf(+ pdfBuffer: Buffer,+ fileSize: number,+): Promise<PDFValidationResult> {+ try {+ if (fileSize > mbToBytes(32)) {+ return {+ isValid: false,+ error: `PDF file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Anthropic's 32MB limit`,+ };+ }++ if (!pdfBuffer || pdfBuffer.length < 5) {+ return {+ isValid: false,+ error: 'Invalid PDF file: too small or corrupted',+ };+ }++ const pdfHeader = pdfBuffer.subarray(0, 5).toString();+ if (!pdfHeader.startsWith('%PDF-')) {+ return {+ isValid: false,+ error: 'Invalid PDF file: missing PDF header',+ };+ }++ const pdfContent = pdfBuffer.toString('binary');+ if (+ pdfContent.includes('/Encrypt ') ||+ pdfContent.includes('/U (') ||+ pdfContent.includes('/O (')+ ) {+ return {+ isValid: false,+ error: 'PDF is password-protected or encrypted. Anthropic requires unencrypted PDFs.',+ };+ }++ const pageMatches = pdfContent.match(/\/Type[\s]*\/Page[^s]/g);+ const estimatedPages = pageMatches ? pageMatches.length : 1;++ if (estimatedPages > 100) {+ return {+ isValid: false,+ error: `PDF has approximately ${estimatedPages} pages, exceeding Anthropic's 100-page limit`,+ };+ }++ return { isValid: true };+ } catch (error) {+ console.error('PDF validation error:', error);+ return {+ isValid: false,+ error: 'Failed to validate PDF file',+ };+ }+}++async function validateOpenAIPdf(fileSize: number): Promise<PDFValidationResult> {+ if (fileSize > 10 * 1024 * 1024) {+ return {+ isValid: false,+ error: "PDF file size exceeds OpenAI's 10MB limit",+ };+ }++ return { isValid: true };+}++async function validateGooglePdf(fileSize: number): Promise<PDFValidationResult> {+ if (fileSize > 20 * 1024 * 1024) {+ return {+ isValid: false,+ error: "PDF file size exceeds Google's 20MB limit",+ };+ }++ return { isValid: true };+}++/**+ * Validates video files for different providers+ * @param videoBuffer - The video file as a buffer+ * @param fileSize - The file size in bytes+ * @param provider - The provider to validate for+ * @returns Promise that resolves to validation result+ */+export async function validateVideo(+ videoBuffer: Buffer,+ fileSize: number,+ provider: Providers,+): Promise<VideoValidationResult> {+ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ if (fileSize > 20 * 1024 * 1024) {+ return {+ isValid: false,+ error: `Video file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,+ };+ }+ }++ if (!videoBuffer || videoBuffer.length < 10) {+ return {+ isValid: false,+ error: 'Invalid video file: too small or corrupted',+ };+ }++ return { isValid: true };+}++/**+ * Validates audio files for different providers+ * @param audioBuffer - The audio file as a buffer+ * @param fileSize - The file size in bytes+ * @param provider - The provider to validate for+ * @returns Promise that resolves to validation result+ */+export async function validateAudio(+ audioBuffer: Buffer,+ fileSize: number,+ provider: Providers,+): Promise<AudioValidationResult> {+ if (provider === Providers.GOOGLE || provider === Providers.VERTEXAI) {+ if (fileSize > 20 * 1024 * 1024) {+ return {+ isValid: false,+ error: `Audio file size (${Math.round(fileSize / (1024 * 1024))}MB) exceeds Google's 20MB limit`,+ };+ }+ }++ if (!audioBuffer || audioBuffer.length < 10) {+ return {+ isValid: false,+ error: 'Invalid audio file: too small or corrupted',+ };+ }++ return { isValid: true };+}diff --git a/packages/api/src/types/files.ts b/packages/api/src/types/files.tsindex 4bfcc23e46..dc37410050 100644--- a/packages/api/src/types/files.ts+++ b/packages/api/src/types/files.ts@@ -1,4 +1,7 @@+import type { IMongoFile } from '@librechat/data-schemas';import type { ServerRequest } from './http';+import type { Readable } from 'stream';+import type { Request } from 'express';export interface STTService {getInstance(): Promise<STTService>;getProviderSchema(req: ServerRequest): Promise<[string, object]>;@@ -26,3 +29,85 @@ export interface AudioProcessingResult {text: string;bytes: number;}++export interface VideoResult {+ videos: Array<{+ type: string;+ mimeType: string;+ data: string;+ }>;+ files: Array<{+ file_id?: string;+ temp_file_id?: string;+ filepath: string;+ source?: string;+ filename: string;+ type: string;+ }>;+}++export interface DocumentResult {+ documents: Array<{+ type: 'document' | 'file' | 'input_file';+ /** Anthropic File Format, `document` */+ source?: {+ type: string;+ media_type: string;+ data: string;+ };+ cache_control?: { type: string };+ citations?: { enabled: boolean };+ /** Google File Format, `document` */+ mimeType?: string;+ data?: string;+ /** OpenAI File Format, `file` */+ file?: {+ filename?: string;+ file_data?: string;+ };+ /** OpenAI Responses API File Format, `input_file` */+ filename?: string;+ file_data?: string;+ }>;+ files: Array<{+ file_id?: string;+ temp_file_id?: string;+ filepath: string;+ source?: string;+ filename: string;+ type: string;+ }>;+}++export interface AudioResult {+ audios: Array<{+ type: string;+ mimeType: string;+ data: string;+ }>;+ files: Array<{+ file_id?: string;+ temp_file_id?: string;+ filepath: string;+ source?: string;+ filename: string;+ type: string;+ }>;+}++export interface ProcessedFile {+ file: IMongoFile;+ content: string;+ metadata: {+ file_id: string;+ temp_file_id?: string;+ filepath: string;+ source?: string;+ filename: string;+ type: string;+ };+}++export interface StrategyFunctions {+ getDownloadStream: (req: Request, filepath: string) => Promise<Readable>;+}diff --git a/packages/client/src/svgs/AudioPaths.tsx b/packages/client/src/svgs/AudioPaths.tsxnew file mode 100644index 0000000000..874f54328d--- /dev/null+++ b/packages/client/src/svgs/AudioPaths.tsx@@ -0,0 +1,41 @@+export default function AudioPaths() {+ return (+ <>+ <path+ d="M8 15v6"+ stroke="white"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <path+ d="M13 8v20"+ stroke="white"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <path+ d="M18 10v16"+ stroke="white"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <path+ d="M23 6v24"+ stroke="white"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ <path+ d="M28 12v12"+ stroke="white"+ strokeWidth="2"+ strokeLinecap="round"+ strokeLinejoin="round"+ />+ </>+ );+}diff --git a/packages/client/src/svgs/VideoPaths.tsx b/packages/client/src/svgs/VideoPaths.tsxnew file mode 100644index 0000000000..6876824e42--- /dev/null+++ b/packages/client/src/svgs/VideoPaths.tsx@@ -0,0 +1,10 @@+export default function VideoPaths() {+ return (+ <>+ {/* Video container - rounded rectangle (not filled) */}+ <rect x="8" y="10" width="20" height="16" rx="3" stroke="white" strokeWidth="2" fill="none" />+ {/* Play button - centered and pointing right */}+ <path d="M22 18l-6 4v-8L22 18z" fill="white" />+ </>+ );+}diff --git a/packages/client/src/svgs/index.ts b/packages/client/src/svgs/index.tsindex 13a5a1cc0a..d3f8c6e45b 100644--- a/packages/client/src/svgs/index.ts+++ b/packages/client/src/svgs/index.ts@@ -65,9 +65,11 @@ export { default as PersonalizationIcon } from './PersonalizationIcon';export { default as MCPIcon } from './MCPIcon';export { default as VectorIcon } from './VectorIcon';export { default as SquirclePlusIcon } from './SquirclePlusIcon';+export { default as AudioPaths } from './AudioPaths';export { default as CodePaths } from './CodePaths';export { default as FileIcon } from './FileIcon';export { default as FilePaths } from './FilePaths';export { default as SheetPaths } from './SheetPaths';export { default as TextPaths } from './TextPaths';+export { default as VideoPaths } from './VideoPaths';export { default as SharePointIcon } from './SharePointIcon';diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.tsindex 75c403afbc..3ad24da67a 100644--- a/packages/data-provider/src/file-config.ts+++ b/packages/data-provider/src/file-config.ts@@ -57,6 +57,27 @@ export const fullMimeTypesList = ['application/zip','image/svg','image/svg+xml',+ // Video formats+ 'video/mp4',+ 'video/avi',+ 'video/mov',+ 'video/wmv',+ 'video/flv',+ 'video/webm',+ 'video/mkv',+ 'video/m4v',+ 'video/3gp',+ 'video/ogv',+ // Audio formats+ 'audio/mp3',+ 'audio/wav',+ 'audio/ogg',+ 'audio/m4a',+ 'audio/aac',+ 'audio/flac',+ 'audio/wma',+ 'audio/opus',+ 'audio/mpeg',...excelFileTypes,];@@ -123,7 +144,9 @@ export const applicationMimeTypes =export const imageMimeTypes = /^image\/(jpeg|gif|png|webp|heic|heif)$/;export const audioMimeTypes =- /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|x-m4a|flac|x-flac|webm)$/;+ /^audio\/(mp3|mpeg|mpeg3|wav|wave|x-wav|ogg|vorbis|mp4|m4a|x-m4a|flac|x-flac|webm|aac|wma|opus)$/;++export const videoMimeTypes = /^video\/(mp4|avi|mov|wmv|flv|webm|mkv|m4v|3gp|ogv)$/;export const defaultOCRMimeTypes = [imageMimeTypes,@@ -142,8 +165,9 @@ export const supportedMimeTypes = [excelMimeTypes,applicationMimeTypes,imageMimeTypes,+ videoMimeTypes,audioMimeTypes,- /** Supported by LC Code Interpreter PAI */+ /** Supported by LC Code Interpreter API *//^image\/(svg|svg\+xml)$/,];@@ -199,6 +223,13 @@ export const fileConfig = {[EModelEndpoint.assistants]: assistantsFileConfig,[EModelEndpoint.azureAssistants]: assistantsFileConfig,[EModelEndpoint.agents]: assistantsFileConfig,+ [EModelEndpoint.anthropic]: {+ fileLimit: 10,+ fileSizeLimit: defaultSizeLimit,+ totalSizeLimit: defaultSizeLimit,+ supportedMimeTypes,+ disabled: false,+ },default: {fileLimit: 10,fileSizeLimit: defaultSizeLimit,diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.tsindex 7d4016449a..61616a57a8 100644--- a/packages/data-provider/src/parsers.ts+++ b/packages/data-provider/src/parsers.ts@@ -369,7 +369,7 @@ export function parseTextParts(continue;}if (part.type === ContentTypes.TEXT) {- const textValue = typeof part.text === 'string' ? part.text : part.text.value;+ const textValue = (typeof part.text === 'string' ? part.text : part.text?.value) || '';if (result.length > 0 &&diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.tsindex 8ae503ef25..a279f4f84d 100644--- a/packages/data-provider/src/schemas.ts+++ b/packages/data-provider/src/schemas.ts@@ -31,6 +31,61 @@ export enum EModelEndpoint {gptPlugins = 'gptPlugins',}+/** Mirrors `@librechat/agents` providers */+export enum Providers {+ OPENAI = 'openAI',+ ANTHROPIC = 'anthropic',+ AZURE = 'azureOpenAI',+ GOOGLE = 'google',+ VERTEXAI = 'vertexai',+ BEDROCK = 'bedrock',+ BEDROCK_LEGACY = 'bedrock_legacy',+ MISTRALAI = 'mistralai',+ MISTRAL = 'mistral',+ OLLAMA = 'ollama',+ DEEPSEEK = 'deepseek',+ OPENROUTER = 'openrouter',+ XAI = 'xai',+}++/**+ * Endpoints that support direct PDF processing in the agent system+ */+export const documentSupportedProviders = new Set<string>([+ EModelEndpoint.anthropic,+ EModelEndpoint.openAI,+ EModelEndpoint.custom,+ EModelEndpoint.azureOpenAI,+ EModelEndpoint.google,+ Providers.VERTEXAI,+ Providers.MISTRALAI,+ Providers.MISTRAL,+ Providers.OLLAMA,+ Providers.DEEPSEEK,+ Providers.OPENROUTER,+ Providers.XAI,+]);++const openAILikeProviders = new Set<string>([+ Providers.OPENAI,+ Providers.AZURE,+ EModelEndpoint.custom,+ Providers.MISTRALAI,+ Providers.MISTRAL,+ Providers.OLLAMA,+ Providers.DEEPSEEK,+ Providers.OPENROUTER,+ Providers.XAI,+]);++export const isOpenAILikeProvider = (provider?: string | null): boolean => {+ return openAILikeProviders.has(provider ?? '');+};++export const isDocumentSupportedProvider = (provider?: string | null): boolean => {+ return documentSupportedProviders.has(provider ?? '');+};+export const paramEndpoints = new Set<EModelEndpoint | string>([EModelEndpoint.agents,EModelEndpoint.openAI,diff --git a/packages/data-provider/src/types/assistants.ts b/packages/data-provider/src/types/assistants.tsindex beb1e10701..246c60a5c3 100644--- a/packages/data-provider/src/types/assistants.ts+++ b/packages/data-provider/src/types/assistants.ts@@ -475,10 +475,20 @@ export type ContentPart = () &PartMetadata;+export type TextData = (Text & PartMetadata) | undefined;+export type TMessageContentParts =- | { type: ContentTypes.ERROR; text?: string | (Text & PartMetadata); error?: string }- | { type: ContentTypes.THINK; think: string | (Text & PartMetadata) }- | { type: ContentTypes.TEXT; text: string | (Text & PartMetadata); tool_call_ids?: string[] }+ | {+ type: ContentTypes.ERROR;+ text?: string | TextData;+ error?: string;+ }+ | { type: ContentTypes.THINK; think?: string | TextData }+ | {+ type: ContentTypes.TEXT;+ text?: string | TextData;+ tool_call_ids?: string[];+ }| {type: ContentTypes.TOOL_CALL;tool_call: (