Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔧 fix: S3 Download Stream with Key Extraction and Blob Storage Encoding for Vision (#6557)
Danny Avilaea2cbc55a747
diff --git a/api/server/services/Files/S3/crud.js b/api/server/services/Files/S3/crud.jsindex 06f9116b69..a9ddcf455e 100644--- a/api/server/services/Files/S3/crud.js+++ b/api/server/services/Files/S3/crud.js@@ -135,20 +135,41 @@ async function uploadFileToS3({ req, file, file_id, basePath = defaultBasePath }}}+/**+ * Extracts the S3 key from a full S3 URL.+ *+ * @param {string} s3Url - The full S3 URL+ * @returns {string} The S3 key+ */+function extractKeyFromS3Url(s3Url) {+ try {+ // Parse the URL+ const url = new URL(s3Url);+ // Extract the path from the URL, removing the leading slash+ let key = url.pathname.substring(1);++ return key;+ } catch (error) {+ throw new Error(`Failed to extract key from S3 URL: ${error.message}`);+ }+}+/*** Retrieves a readable stream for a file stored in S3.*+ * @param {ServerRequest} req - Server request object.* @param {string} filePath - The S3 key of the file.* @returns {Promise<NodeJS.ReadableStream>}*/-async function getS3FileStream(filePath) {- const params = { Bucket: bucketName, Key: filePath };+async function getS3FileStream(_req, filePath) {try {+ const Key = extractKeyFromS3Url(filePath);+ const params = { Bucket: bucketName, Key };const s3 = initializeS3();const data = await s3.send(new GetObjectCommand(params));return data.Body; // Returns a Node.js ReadableStream.} catch (error) {- logger.error('[getS3FileStream] Error retrieving S3 file stream:', error.message);+ logger.error('[getS3FileStream] Error retrieving S3 file stream:', error);throw error;}}diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.jsindex 707632fb6a..a23ce4cf2c 100644--- a/api/server/services/Files/images/encode.js+++ b/api/server/services/Files/images/encode.js@@ -37,17 +37,21 @@ const base64Only = new Set([EModelEndpoint.bedrock,]);+const blobStorageSources = new Set([FileSources.azure, FileSources.s3]);+/*** Encodes and formats the given files.* @param {Express.Request} req - The request object.* @param {Array<MongoFile>} files - The array of files to encode and format.* @param {EModelEndpoint} [endpoint] - Optional: The endpoint for the image.* @param {string} [mode] - Optional: The endpoint mode for the image.- * @returns {Promise<Object>} - A promise that resolves to the result object containing the encoded images and file details.+ * @returns {Promise<{ text: string; files: MongoFile[]; image_urls: MessageContentImageUrl[] }>} - A promise that resolves to the result object containing the encoded images and file details.*/async function encodeAndFormat(req, files, endpoint, mode) {const promises = [];+ /** @type {Record<FileSources, Pick<ReturnType<typeof getStrategyFunctions>, 'prepareImagePayload' | 'getDownloadStream'>>} */const encodingMethods = {};+ /** @type {{ text: string; files: MongoFile[]; image_urls: MessageContentImageUrl[] }} */const result = {text: '',files: [],@@ -59,6 +63,7 @@ async function encodeAndFormat(req, files, endpoint, mode) {}for (let file of files) {+ /** @type {FileSources} */const source = file.source ?? FileSources.local;if (source === FileSources.text && file.text) {result.text += `${!result.text ? 'Attached document(s):\n```md' : '\n\n---\n\n'}# "${file.filename}"\n${file.text}\n`;@@ -70,18 +75,51 @@ async function encodeAndFormat(req, files, endpoint, mode) {}if (!encodingMethods[source]) {- const { prepareImagePayload } = getStrategyFunctions(source);+ const { prepareImagePayload, getDownloadStream } = getStrategyFunctions(source);if (!prepareImagePayload) {throw new Error(`Encoding function not implemented for ${source}`);}- encodingMethods[source] = prepareImagePayload;+ encodingMethods[source] = { prepareImagePayload, getDownloadStream };}- const preparePayload = encodingMethods[source];+ const preparePayload = encodingMethods[source].prepareImagePayload;+ /* We need to fetch the image and convert it to base64 if we are using S3/Azure Blob storage. */+ if (blobStorageSources.has(source)) {+ try {+ const downloadStream = encodingMethods[source].getDownloadStream;+ const stream = await downloadStream(req, file.filepath);+ const streamPromise = new Promise((resolve, reject) => {+ /** @type {Uint8Array[]} */+ const chunks = [];+ stream.on('readable', () => {+ let chunk;+ while (null !== (chunk = stream.read())) {+ chunks.push(chunk);+ }+ });++ stream.on('end', () => {+ const buffer = Buffer.concat(chunks);+ const base64Data = buffer.toString('base64');+ resolve(base64Data);+ });+ stream.on('error', (error) => {+ reject(error);+ });+ });+ const base64Data = await streamPromise;+ promises.push([file, base64Data]);+ } catch (error) {+ logger.error(+ `Error processing blob storage file stream for ${file.name} base64 payload:`,+ error,+ );+ continue;+ }- /* Google & Anthropic don't support passing URLs to payload */- if (source !== FileSources.local && base64Only.has(endpoint)) {+ /* Google & Anthropic don't support passing URLs to payload */+ } else if (source !== FileSources.local && base64Only.has(endpoint)) {const [_file, imageURL] = await preparePayload(req, file);promises.push([_file, await fetchImageToBase64(imageURL)]);continue;diff --git a/api/typedefs.js b/api/typedefs.jsindex 21c4f1fecc..74d9ef5209 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -403,6 +403,12 @@* @memberof typedefs*/+/**+ * @exports MessageContentImageUrl+ * @typedef {import('librechat-data-provider').Agents.MessageContentImageUrl} MessageContentImageUrl+ * @memberof typedefs+ */+/** Prompts *//*** @exports TPrompt@@ -759,6 +765,11 @@* @typedef {import('mongoose').Schema} MongooseSchema* @memberof typedefs*/+/**+ * @exports MongoFile+ * @typedef {import('@librechat/data-schemas').IMongoFile} MongoFile+ * @memberof typedefs+ *//*** @exports ObjectId