Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)
Danny Avilad20970f5c5c9
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.jsindex 5b7ea103c2..aa39084b9f 100644--- a/api/app/clients/BaseClient.js+++ b/api/app/clients/BaseClient.js@@ -46,6 +46,10 @@ class BaseClient {logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);}+ async addPreviousAttachments(messages) {+ return messages;+ }+async recordTokenUsage({ promptTokens, completionTokens }) {logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {promptTokens,@@ -484,20 +488,22 @@ class BaseClient {mapMethod = this.getMessageMapMethod();}- const orderedMessages = this.constructor.getMessagesForConversation({+ let _messages = this.constructor.getMessagesForConversation({messages,parentMessageId,mapMethod,});+ _messages = await this.addPreviousAttachments(_messages);+if (!this.shouldSummarize) {- return orderedMessages;+ return _messages;}// Find the latest message with a 'summary' property- for (let i = orderedMessages.length - 1; i >= 0; i--) {- if (orderedMessages[i]?.summary) {- this.previous_summary = orderedMessages[i];+ for (let i = _messages.length - 1; i >= 0; i--) {+ if (_messages[i]?.summary) {+ this.previous_summary = _messages[i];break;}}@@ -512,7 +518,7 @@ class BaseClient {});}- return orderedMessages;+ return _messages;}async saveMessageToDatabase(message, endpointOptions, user = null) {@@ -618,6 +624,11 @@ class BaseClient {* An additional 3 tokens need to be added for assistant label priming after all messages have been counted.* In our implementation, this is accounted for in the getMessagesWithinTokenLimit method.*+ * The content parts example was adapted from the following example:+ * https://github.com/openai/openai-cookbook/pull/881/files+ *+ * Note: image token calculation is to be done elsewhere where we have access to the image metadata+ ** @param {Object} message*/getTokenCountForMessage(message) {@@ -631,11 +642,18 @@ class BaseClient {}const processValue = (value) => {- if (typeof value === 'object' && value !== null) {- for (let [nestedKey, nestedValue] of Object.entries(value)) {- if (nestedKey === 'image_url' || nestedValue === 'image_url') {+ if (Array.isArray(value)) {+ for (let item of value) {+ if (!item || !item.type || item.type === 'image_url') {continue;}++ const nestedValue = item[item.type];++ if (!nestedValue) {+ continue;+ }+processValue(nestedValue);}} else {diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.jsindex 1c22d6f7d4..f832b3ca13 100644--- a/api/app/clients/OpenAIClient.js+++ b/api/app/clients/OpenAIClient.js@@ -1,6 +1,6 @@const OpenAI = require('openai');const { HttpsProxyAgent } = require('https-proxy-agent');-const { getResponseSender } = require('librechat-data-provider');+const { getResponseSender, ImageDetailCost, ImageDetail } = require('librechat-data-provider');const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('~/utils');@@ -8,8 +8,9 @@ const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');const { handleOpenAIErrors } = require('./tools/util');const spendTokens = require('~/models/spendTokens');const { createLLM, RunManager } = require('./llm');-const { isEnabled } = require('~/server/utils');const ChatGPTClient = require('./ChatGPTClient');+const { isEnabled } = require('~/server/utils');+const { getFiles } = require('~/models/File');const { summaryBuffer } = require('./memory');const { runTitleChain } = require('./chains');const { tokenSplit } = require('./document');@@ -76,16 +77,7 @@ class OpenAIClient extends BaseClient {};}- this.isVisionModel = validateVisionModel(this.modelOptions.model);-- if (this.options.attachments && !this.isVisionModel) {- this.modelOptions.model = 'gpt-4-vision-preview';- this.isVisionModel = true;- }-- if (this.isVisionModel) {- delete this.modelOptions.stop;- }+ this.checkVisionRequest(this.options.attachments);const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};if (OPENROUTER_API_KEY && !this.azure) {@@ -204,6 +196,27 @@ class OpenAIClient extends BaseClient {return this;}+ /**+ *+ * Checks if the model is a vision model based on request attachments and sets the appropriate options:+ * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.+ * - Sets `this.isVisionModel` to `true` if vision request.+ * - Deletes `this.modelOptions.stop` if vision request.+ * @param {Array<Promise<MongoFile[]> | MongoFile[]> | Record<string, MongoFile[]>} attachments+ */+ checkVisionRequest(attachments) {+ this.isVisionModel = validateVisionModel(this.modelOptions.model);++ if (attachments && !this.isVisionModel) {+ this.modelOptions.model = 'gpt-4-vision-preview';+ this.isVisionModel = true;+ }++ if (this.isVisionModel) {+ delete this.modelOptions.stop;+ }+ }+setupTokens() {if (this.isChatCompletion) {this.startToken = '||>';@@ -288,7 +301,11 @@ class OpenAIClient extends BaseClient {tokenizerCallsCount++;}- // Returns the token count of a given text. It also checks and resets the tokenizers if necessary.+ /**+ * Returns the token count of a given text. It also checks and resets the tokenizers if necessary.+ * @param {string} text - The text to get the token count for.+ * @returns {number} The token count of the given text.+ */getTokenCount(text) {this.resetTokenizersIfNecessary();try {@@ -301,10 +318,33 @@ class OpenAIClient extends BaseClient {}}+ /**+ * Calculate the token cost for an image based on its dimensions and detail level.+ *+ * @param {Object} image - The image object.+ * @param {number} image.width - The width of the image.+ * @param {number} image.height - The height of the image.+ * @param {'low'|'high'|string|undefined} [image.detail] - The detail level ('low', 'high', or other).+ * @returns {number} The calculated token cost.+ */+ calculateImageTokenCost({ width, height, detail }) {+ if (detail === 'low') {+ return ImageDetailCost.LOW;+ }++ // Calculate the number of 512px squares+ const numSquares = Math.ceil(width / 512) * Math.ceil(height / 512);++ // Default to high detail cost calculation+ return numSquares * ImageDetailCost.HIGH + ImageDetailCost.ADDITIONAL;+ }+getSaveOptions() {return {chatGptLabel: this.options.chatGptLabel,promptPrefix: this.options.promptPrefix,+ resendImages: this.options.resendImages,+ imageDetail: this.options.imageDetail,...this.modelOptions,};}@@ -317,6 +357,69 @@ class OpenAIClient extends BaseClient {};}+ /**+ *+ * @param {TMessage[]} _messages+ * @returns {TMessage[]}+ */+ async addPreviousAttachments(_messages) {+ if (!this.options.resendImages) {+ return _messages;+ }++ /**+ *+ * @param {TMessage} message+ */+ const processMessage = async (message) => {+ if (!this.message_file_map) {+ /** @type {Record<string, MongoFile[]> */+ this.message_file_map = {};+ }++ const fileIds = message.files.map((file) => file.file_id);+ const files = await getFiles({+ file_id: { $in: fileIds },+ });++ await this.addImageURLs(message, files);++ this.message_file_map[message.messageId] = files;+ return message;+ };++ const promises = [];++ for (const message of _messages) {+ if (!message.files) {+ promises.push(message);+ continue;+ }++ promises.push(processMessage(message));+ }++ const messages = await Promise.all(promises);++ this.checkVisionRequest(this.message_file_map);+ return messages;+ }++ /**+ *+ * Adds image URLs to the message object and returns the files+ *+ * @param {TMessage[]} messages+ * @param {MongoFile[]} files+ * @returns {Promise<MongoFile[]>}+ */+ async addImageURLs(message, attachments) {+ const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);++ message.image_urls = image_urls;+ return files;+ }+async buildMessages(messages,parentMessageId,@@ -355,13 +458,23 @@ class OpenAIClient extends BaseClient {}if (this.options.attachments) {- const attachments = await this.options.attachments;- const { files, image_urls } = await encodeAndFormat(- this.options.req,- attachments.filter((file) => file.type.includes('image')),+ const attachments = (await this.options.attachments).filter((file) =>+ file.type.includes('image'),+ );++ if (this.message_file_map) {+ this.message_file_map[orderedMessages[orderedMessages.length - 1].messageId] = attachments;+ } else {+ this.message_file_map = {+ [orderedMessages[orderedMessages.length - 1].messageId]: attachments,+ };+ }++ const files = await this.addImageURLs(+ orderedMessages[orderedMessages.length - 1],+ attachments,);- orderedMessages[orderedMessages.length - 1].image_urls = image_urls;this.options.attachments = files;}@@ -372,10 +485,25 @@ class OpenAIClient extends BaseClient {assistantName: this.options?.chatGptLabel,});- if (this.contextStrategy && !orderedMessages[i].tokenCount) {+ const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;++ /* If tokens were never counted, or, is a Vision request and the message has files, count again */+ if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);}+ /* If message has files, calculate image token cost */+ if (this.message_file_map && this.message_file_map[message.messageId]) {+ const attachments = this.message_file_map[message.messageId];+ for (const file of attachments) {+ orderedMessages[i].tokenCount += this.calculateImageTokenCost({+ width: file.width,+ height: file.height,+ detail: this.options.imageDetail ?? ImageDetail.auto,+ });+ }+ }+return formattedMessage;});@@ -780,7 +908,6 @@ ${convo}if (this.isChatCompletion) {modelOptions.messages = payload;} else {- // TODO: unreachable code. Need to implement completions call for non-chat modelsmodelOptions.prompt = payload;}@@ -916,6 +1043,8 @@ ${convo}clientOptions.addMetadata({ finish_reason });}+ logger.debug('[OpenAIClient] chatCompletion response', chatCompletion);+return message.content;} catch (err) {if (diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.jsindex f26df8a2d1..6118c3547a 100644--- a/api/app/clients/PluginsClient.js+++ b/api/app/clients/PluginsClient.js@@ -112,7 +112,7 @@ class PluginsClient extends OpenAIClient {signal: this.abortController.signal,openAIApiKey: this.openAIApiKey,conversationId: this.conversationId,- debug: this.options?.debug,+ fileStrategy: this.options.req.app.locals.fileStrategy,message,},});diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.jsindex a31d82fe12..8c2226215c 100644--- a/api/app/clients/specs/OpenAIClient.test.js+++ b/api/app/clients/specs/OpenAIClient.test.js@@ -546,6 +546,39 @@ describe('OpenAIClient', () => {expect(totalTokens).toBe(testCase.expected);});});++ const vision_request = [+ {+ role: 'user',+ content: [+ {+ type: 'text',+ text: 'describe what is in this image?',+ },+ {+ type: 'image_url',+ image_url: {+ url: 'https://venturebeat.com/wp-content/uploads/2019/03/openai-1.png',+ detail: 'high',+ },+ },+ ],+ },+ ];++ const expectedTokens = 14;+ const visionModel = 'gpt-4-vision-preview';++ it(`should return ${expectedTokens} tokens for model ${visionModel} (Vision Request)`, () => {+ client.modelOptions.model = visionModel;+ client.selectTokenizer();+ // 3 tokens for assistant label+ let totalTokens = 3;+ for (let message of vision_request) {+ totalTokens += client.getTokenCountForMessage(message);+ }+ expect(totalTokens).toBe(expectedTokens);+ });});describe('sendMessage/getCompletion/chatCompletion', () => {diff --git a/api/app/clients/tools/DALL-E.js b/api/app/clients/tools/DALL-E.jsindex 387294a1cb..7f0c952030 100644--- a/api/app/clients/tools/DALL-E.js+++ b/api/app/clients/tools/DALL-E.js@@ -1,20 +1,13 @@// From https://platform.openai.com/docs/api-reference/images/create// To use this tool, you must pass in a configured OpenAIApi object.-const fs = require('fs');-const path = require('path');const OpenAI = require('openai');// const { genAzureEndpoint } = require('~/utils/genAzureEndpoints');const { v4: uuidv4 } = require('uuid');const { Tool } = require('langchain/tools');const { HttpsProxyAgent } = require('https-proxy-agent');-const {- saveImageToFirebaseStorage,- getFirebaseStorageImageUrl,- getFirebaseStorage,-} = require('~/server/services/Files/Firebase');const { getImageBasename } = require('~/server/services/Files/images');+const { processFileURL } = require('~/server/services/Files/process');const extractBaseURL = require('~/utils/extractBaseURL');-const saveImageFromUrl = require('./saveImageFromUrl');const { logger } = require('~/config');const { DALLE_REVERSE_PROXY, PROXY } = process.env;@@ -23,6 +16,7 @@ class OpenAICreateImage extends Tool {super();this.userId = fields.userId;+ this.fileStrategy = fields.fileStrategy;let apiKey = fields.DALLE_API_KEY || this.getApiKey();const config = { apiKey };@@ -82,12 +76,8 @@ Guidelines:.trim();}- getMarkdownImageUrl(imageName) {- const imageUrl = path- .join(this.relativeImageUrl, imageName)- .replace(/\\/g, '/')- .replace('public/', '');- return ``;+ wrapInMarkdown(imageUrl) {+ return ``;}async _call(input) {@@ -118,45 +108,21 @@ Guidelines:});}- this.outputPath = path.resolve(- __dirname,- '..',- '..',- '..',- '..',- 'client',- 'public',- 'images',- this.userId,- );-- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');- this.relativeImageUrl = path.relative(appRoot, this.outputPath);-- // Check if directory exists, if not create it- if (!fs.existsSync(this.outputPath)) {- fs.mkdirSync(this.outputPath, { recursive: true });- }+ try {+ const result = await processFileURL({+ fileStrategy: this.fileStrategy,+ userId: this.userId,+ URL: theImageUrl,+ fileName: imageName,+ basePath: 'images',+ });- const storage = getFirebaseStorage();- if (storage) {- try {- await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);- this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);- logger.debug('[DALL-E] result: ' + this.result);- } catch (error) {- logger.error('Error while saving the image to Firebase Storage:', error);- this.result = `Failed to save the image to Firebase Storage. ${error.message}`;- }- } else {- try {- await saveImageFromUrl(theImageUrl, this.outputPath, imageName);- this.result = this.getMarkdownImageUrl(imageName);- } catch (error) {- logger.error('Error while saving the image locally:', error);- this.result = `Failed to save the image locally. ${error.message}`;- }+ this.result = this.wrapInMarkdown(result);+ } catch (error) {+ logger.error('Error while saving the image:', error);+ this.result = `Failed to save the image locally. ${error.message}`;}+return this.result;}}diff --git a/api/app/clients/tools/saveImageFromUrl.js b/api/app/clients/tools/saveImageFromUrl.jsdeleted file mode 100644index b9fbcc895b..0000000000--- a/api/app/clients/tools/saveImageFromUrl.js+++ /dev/null@@ -1,46 +0,0 @@-const fs = require('fs');-const path = require('path');-const axios = require('axios');-const { logger } = require('~/config');--async function saveImageFromUrl(url, outputPath, outputFilename) {- try {- // Fetch the image from the URL- const response = await axios({- url,- responseType: 'stream',- });-- // Get the content type from the response headers- const contentType = response.headers['content-type'];- let extension = contentType.split('/').pop();-- // Check if the output directory exists, if not, create it- if (!fs.existsSync(outputPath)) {- fs.mkdirSync(outputPath, { recursive: true });- }-- // Replace or append the correct extension- const extRegExp = new RegExp(path.extname(outputFilename) + '$');- outputFilename = outputFilename.replace(extRegExp, `.${extension}`);- if (!path.extname(outputFilename)) {- outputFilename += `.${extension}`;- }-- // Create a writable stream for the output path- const outputFilePath = path.join(outputPath, outputFilename);- const writer = fs.createWriteStream(outputFilePath);-- // Pipe the response data to the output file- response.data.pipe(writer);-- return new Promise((resolve, reject) => {- writer.on('finish', resolve);- writer.on('error', reject);- });- } catch (error) {- logger.error('[saveImageFromUrl] Error while saving the image:', error);- }-}--module.exports = saveImageFromUrl;diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.jsindex 17d0368f39..16326acdee 100644--- a/api/app/clients/tools/structured/DALLE3.js+++ b/api/app/clients/tools/structured/DALLE3.js@@ -1,20 +1,13 @@// From https://platform.openai.com/docs/guides/images/usage?context=node// To use this tool, you must pass in a configured OpenAIApi object.-const fs = require('fs');-const path = require('path');const { z } = require('zod');const OpenAI = require('openai');const { v4: uuidv4 } = require('uuid');const { Tool } = require('langchain/tools');const { HttpsProxyAgent } = require('https-proxy-agent');-const {- saveImageToFirebaseStorage,- getFirebaseStorageImageUrl,- getFirebaseStorage,-} = require('~/server/services/Files/Firebase');const { getImageBasename } = require('~/server/services/Files/images');+const { processFileURL } = require('~/server/services/Files/process');const extractBaseURL = require('~/utils/extractBaseURL');-const saveImageFromUrl = require('../saveImageFromUrl');const { logger } = require('~/config');const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;@@ -23,6 +16,7 @@ class DALLE3 extends Tool {super();this.userId = fields.userId;+ this.fileStrategy = fields.fileStrategy;let apiKey = fields.DALLE_API_KEY || this.getApiKey();const config = { apiKey };if (DALLE_REVERSE_PROXY) {@@ -91,12 +85,8 @@ class DALLE3 extends Tool {.trim();}- getMarkdownImageUrl(imageName) {- const imageUrl = path- .join(this.relativeImageUrl, imageName)- .replace(/\\/g, '/')- .replace('public/', '');- return ``;+ wrapInMarkdown(imageUrl) {+ return ``;}async _call(data) {@@ -143,43 +133,19 @@ Error Message: ${error.message}`;});}- this.outputPath = path.resolve(- __dirname,- '..',- '..',- '..',- '..',- '..',- 'client',- 'public',- 'images',- this.userId,- );- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');- this.relativeImageUrl = path.relative(appRoot, this.outputPath);-- // Check if directory exists, if not create it- if (!fs.existsSync(this.outputPath)) {- fs.mkdirSync(this.outputPath, { recursive: true });- }- const storage = getFirebaseStorage();- if (storage) {- try {- await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);- this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);- logger.debug('[DALL-E-3] result: ' + this.result);- } catch (error) {- logger.error('Error while saving the image to Firebase Storage:', error);- this.result = `Failed to save the image to Firebase Storage. ${error.message}`;- }- } else {- try {- await saveImageFromUrl(theImageUrl, this.outputPath, imageName);- this.result = this.getMarkdownImageUrl(imageName);- } catch (error) {- logger.error('Error while saving the image locally:', error);- this.result = `Failed to save the image locally. ${error.message}`;- }+ try {+ const result = await processFileURL({+ fileStrategy: this.fileStrategy,+ userId: this.userId,+ URL: theImageUrl,+ fileName: imageName,+ basePath: 'images',+ });++ this.result = this.wrapInMarkdown(result);+ } catch (error) {+ logger.error('Error while saving the image:', error);+ this.result = `Failed to save the image locally. ${error.message}`;}return this.result;diff --git a/api/app/clients/tools/structured/specs/DALLE3.spec.js b/api/app/clients/tools/structured/specs/DALLE3.spec.jsindex 34fa3ebf00..65a5fbd844 100644--- a/api/app/clients/tools/structured/specs/DALLE3.spec.js+++ b/api/app/clients/tools/structured/specs/DALLE3.spec.js@@ -1,20 +1,13 @@-const fs = require('fs');-const path = require('path');const OpenAI = require('openai');const DALLE3 = require('../DALLE3');-const {- getFirebaseStorage,- saveImageToFirebaseStorage,-} = require('~/server/services/Files/Firebase');-const saveImageFromUrl = require('../../saveImageFromUrl');+const { processFileURL } = require('~/server/services/Files/process');+const { logger } = require('~/config');jest.mock('openai');-jest.mock('~/server/services/Files/Firebase', () => ({- getFirebaseStorage: jest.fn(),- saveImageToFirebaseStorage: jest.fn(),- getFirebaseStorageImageUrl: jest.fn(),+jest.mock('~/server/services/Files/process', () => ({+ processFileURL: jest.fn(),}));jest.mock('~/server/services/Files/images', () => ({@@ -50,10 +43,6 @@ jest.mock('fs', () => {};});-jest.mock('../../saveImageFromUrl', () => {- return jest.fn();-});-jest.mock('path', () => {return {resolve: jest.fn(),@@ -99,10 +88,8 @@ describe('DALLE3', () => {it('should generate markdown image URL correctly', () => {const imageName = 'test.png';- path.join.mockReturnValue('images/test.png');- path.relative.mockReturnValue('images/test.png');- const markdownImage = dalle.getMarkdownImageUrl(imageName);- expect(markdownImage).toBe('');+ const markdownImage = dalle.wrapInMarkdown(imageName);+ expect(markdownImage).toBe('');});it('should call OpenAI API with correct parameters', async () => {@@ -122,11 +109,7 @@ describe('DALLE3', () => {};generate.mockResolvedValue(mockResponse);- saveImageFromUrl.mockResolvedValue(true);- fs.existsSync.mockReturnValue(true);- path.resolve.mockReturnValue('/fakepath/images');- path.join.mockReturnValue('/fakepath/images/img-test.png');- path.relative.mockReturnValue('images/img-test.png');+ processFileURL.mockResolvedValue('http://example.com/img-test.png');const result = await dalle._call(mockData);@@ -138,6 +121,7 @@ describe('DALLE3', () => {prompt: mockData.prompt,n: 1,});+expect(result).toContain('![generated image]');});@@ -184,23 +168,6 @@ describe('DALLE3', () => {});});- it('should create the directory if it does not exist', async () => {- const mockData = {- prompt: 'A test prompt',- };- const mockResponse = {- data: [- {- url: 'http://example.com/img-test.png',- },- ],- };- generate.mockResolvedValue(mockResponse);- fs.existsSync.mockReturnValue(false); // Simulate directory does not exist- await dalle._call(mockData);- expect(fs.mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true });- });-it('should log an error and return the image URL if there is an error saving the image', async () => {const mockData = {prompt: 'A test prompt',@@ -214,31 +181,12 @@ describe('DALLE3', () => {};const error = new Error('Error while saving the image');generate.mockResolvedValue(mockResponse);- saveImageFromUrl.mockRejectedValue(error);+ processFileURL.mockRejectedValue(error);const result = await dalle._call(mockData);- expect(logger.error).toHaveBeenCalledWith('Error while saving the image locally:', error);+ expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);expect(result).toBe('Failed to save the image locally. Error while saving the image');});- it('should save image to Firebase Storage if Firebase is initialized', async () => {- const mockData = {- prompt: 'A test prompt',- };- const mockImageUrl = 'http://example.com/img-test.png';- const mockResponse = { data: [{ url: mockImageUrl }] };- generate.mockResolvedValue(mockResponse);- getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized-- await dalle._call(mockData);-- expect(getFirebaseStorage).toHaveBeenCalled();- expect(saveImageToFirebaseStorage).toHaveBeenCalledWith(- undefined,- mockImageUrl,- expect.any(String),- );- });-it('should handle error when saving image to Firebase Storage fails', async () => {const mockData = {prompt: 'A test prompt',@@ -247,17 +195,11 @@ describe('DALLE3', () => {const mockResponse = { data: [{ url: mockImageUrl }] };const error = new Error('Error while saving to Firebase');generate.mockResolvedValue(mockResponse);- getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized- saveImageToFirebaseStorage.mockRejectedValue(error);+ processFileURL.mockRejectedValue(error);const result = await dalle._call(mockData);- expect(logger.error).toHaveBeenCalledWith(- 'Error while saving the image to Firebase Storage:',- error,- );- expect(result).toBe(- 'Failed to save the image to Firebase Storage. Error while saving to Firebase',- );+ expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);+ expect(result).toContain('Failed to save the image');});});diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.jsindex 352dd5dec7..1d9a3a0074 100644--- a/api/app/clients/tools/util/handleTools.js+++ b/api/app/clients/tools/util/handleTools.js@@ -170,6 +170,8 @@ const loadTools = async ({const toolOptions = {serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },+ dalle: { fileStrategy: options.fileStrategy },+ 'dall-e': { fileStrategy: options.fileStrategy },};const toolAuthFields = {};diff --git a/api/cache/keyvRedis.js b/api/cache/keyvRedis.jsindex f723429ee2..9501045e4e 100644--- a/api/cache/keyvRedis.js+++ b/api/cache/keyvRedis.js@@ -10,10 +10,11 @@ if (REDIS_URI && isEnabled(USE_REDIS)) {keyvRedis = new KeyvRedis(REDIS_URI, { useRedisSets: false });keyvRedis.on('error', (err) => logger.error('KeyvRedis connection error:', err));keyvRedis.setMaxListeners(20);-} else {logger.info(- '`REDIS_URI` not provided, or `USE_REDIS` not set. Redis module will not be initialized.',+ '[Optional] Redis initialized. Note: Redis support is experimental. If you have issues, disable it. Cache needs to be flushed for values to refresh.',);+} else {+ logger.info('[Optional] Redis not initialized. Note: Redis support is experimental.');}module.exports = keyvRedis;diff --git a/api/config/paths.js b/api/config/paths.jsindex 2f577a183f..41e3ac5054 100644--- a/api/config/paths.js+++ b/api/config/paths.js@@ -1,6 +1,7 @@const path = require('path');module.exports = {+ dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),};diff --git a/api/models/schema/defaults.js b/api/models/schema/defaults.jsindex feedf1019a..39a6430f46 100644--- a/api/models/schema/defaults.js+++ b/api/models/schema/defaults.js@@ -84,6 +84,12 @@ const conversationPreset = {type: String,// default: null,},+ resendImages: {+ type: Boolean,+ },+ imageDetail: {+ type: String,+ },};const agentOptions = {diff --git a/api/models/schema/fileSchema.js b/api/models/schema/fileSchema.jsindex bf9db4864d..471b7bfd70 100644--- a/api/models/schema/fileSchema.js+++ b/api/models/schema/fileSchema.js@@ -1,3 +1,4 @@+const { FileSources } = require('librechat-data-provider');const mongoose = require('mongoose');/**@@ -12,6 +13,7 @@ const mongoose = require('mongoose');* @property {'file'} object - Type of object, always 'file'* @property {string} type - Type of file* @property {number} usage - Number of uses of the file+ * @property {string} [source] - The source of the file* @property {number} [width] - Optional width of the file* @property {number} [height] - Optional height of the file* @property {Date} [expiresAt] - Optional height of the file@@ -42,11 +44,6 @@ const fileSchema = mongoose.Schema(type: Number,required: true,},- usage: {- type: Number,- required: true,- default: 0,- },filename: {type: String,required: true,@@ -64,6 +61,15 @@ const fileSchema = mongoose.Schema(type: String,required: true,},+ usage: {+ type: Number,+ required: true,+ default: 0,+ },+ source: {+ type: String,+ default: FileSources.local,+ },width: Number,height: Number,expiresAt: {diff --git a/api/server/index.js b/api/server/index.jsindex 3c6429e47d..86806b5914 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -5,33 +5,28 @@ const cors = require('cors');const express = require('express');const passport = require('passport');const mongoSanitize = require('express-mongo-sanitize');-const { initializeFirebase } = require('~/server/services/Files/Firebase/initialize');-const loadCustomConfig = require('~/server/services/Config/loadCustomConfig');-const errorController = require('~/server/controllers/ErrorController');-const configureSocialLogins = require('~/server/socialLogins');-const noIndex = require('~/server/middleware/noIndex');+const errorController = require('./controllers/ErrorController');+const { jwtLogin, passportLogin } = require('~/strategies');+const configureSocialLogins = require('./socialLogins');const { connectDb, indexSync } = require('~/lib/db');+const AppService = require('./services/AppService');+const noIndex = require('./middleware/noIndex');const { logger } = require('~/config');-const routes = require('~/server/routes');-const paths = require('~/config/paths');+const routes = require('./routes');const { PORT, HOST, ALLOW_SOCIAL_LOGIN } = process.env ?? {};const port = Number(PORT) || 3080;const host = HOST || 'localhost';-const projectPath = path.join(__dirname, '..', '..', 'client');-const { jwtLogin, passportLogin } = require('~/strategies');const startServer = async () => {await connectDb();logger.info('Connected to MongoDB');- await loadCustomConfig();- initializeFirebase();await indexSync();const app = express();- app.locals.config = paths;+ await AppService(app);// Middlewareapp.use(noIndex);@@ -39,14 +34,14 @@ const startServer = async () => {app.use(express.json({ limit: '3mb' }));app.use(mongoSanitize());app.use(express.urlencoded({ extended: true, limit: '3mb' }));- app.use(express.static(path.join(projectPath, 'dist')));- app.use(express.static(path.join(projectPath, 'public')));+ app.use(express.static(app.locals.paths.dist));+ app.use(express.static(app.locals.paths.publicPath));app.set('trust proxy', 1); // trust first proxyapp.use(cors());if (!ALLOW_SOCIAL_LOGIN) {console.warn(- 'Social logins are disabled. Set Envrionment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',+ 'Social logins are disabled. Set Environment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',);}@@ -81,7 +76,7 @@ const startServer = async () => {app.use('/api/files', routes.files);app.use((req, res) => {- res.status(404).sendFile(path.join(projectPath, 'dist', 'index.html'));+ res.status(404).sendFile(path.join(app.locals.paths.dist, 'index.html'));});app.listen(port, host, () => {diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.jsindex 543815e367..91d0caceaa 100644--- a/api/server/middleware/buildEndpointOption.js+++ b/api/server/middleware/buildEndpointOption.js@@ -1,10 +1,10 @@-const { processFiles } = require('~/server/services/Files');+const { parseConvo, EModelEndpoint } = require('librechat-data-provider');+const { processFiles } = require('~/server/services/Files/process');+const gptPlugins = require('~/server/services/Endpoints/gptPlugins');+const anthropic = require('~/server/services/Endpoints/anthropic');const openAI = require('~/server/services/Endpoints/openAI');const custom = require('~/server/services/Endpoints/custom');const google = require('~/server/services/Endpoints/google');-const anthropic = require('~/server/services/Endpoints/anthropic');-const gptPlugins = require('~/server/services/Endpoints/gptPlugins');-const { parseConvo, EModelEndpoint } = require('librechat-data-provider');const buildFunction = {[EModelEndpoint.openAI]: openAI.buildOptions,diff --git a/api/server/routes/files/avatar.js b/api/server/routes/files/avatar.jsindex a7bb07c0f9..5abba85f9e 100644--- a/api/server/routes/files/avatar.js+++ b/api/server/routes/files/avatar.js@@ -1,7 +1,7 @@const express = require('express');const multer = require('multer');-const uploadAvatar = require('~/server/services/Files/images/avatar/uploadAvatar');+const uploadAvatar = require('~/server/services/Files/images/avatar');const { requireJwtAuth } = require('~/server/middleware/');const User = require('~/models/User');@@ -23,7 +23,12 @@ router.post('/', requireJwtAuth, upload.single('input'), async (req, res) => {if (!user) {throw new Error('User not found');}- const url = await uploadAvatar(userId, input, manual);+ const url = await uploadAvatar({+ input,+ userId,+ manual,+ fileStrategy: req.app.locals.fileStrategy,+ });res.json({ url });} catch (error) {diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.jsindex d9df1bdd75..3fea2e5d07 100644--- a/api/server/routes/files/files.js+++ b/api/server/routes/files/files.js@@ -1,36 +1,29 @@const { z } = require('zod');-const path = require('path');-const fs = require('fs').promises;const express = require('express');-const { deleteFiles } = require('~/models');+const { FileSources } = require('librechat-data-provider');+const { getStrategyFunctions } = require('~/server/services/Files/strategies');+const { deleteFiles, getFiles } = require('~/models');const { logger } = require('~/config');const router = express.Router();const isUUID = z.string().uuid();-const isValidPath = (req, base, subfolder, filepath) => {- const normalizedBase = path.resolve(base, subfolder, req.user.id);- const normalizedFilepath = path.resolve(filepath);- return normalizedFilepath.startsWith(normalizedBase);-};--const deleteFile = async (req, file) => {- const { publicPath } = req.app.locals.config;- const parts = file.filepath.split(path.sep);- const subfolder = parts[1];- const filepath = path.join(publicPath, file.filepath);-- if (!isValidPath(req, publicPath, subfolder, filepath)) {- throw new Error('Invalid file path');+router.get('/', async (req, res) => {+ try {+ const files = await getFiles({ user: req.user.id });+ res.status(200).send(files);+ } catch (error) {+ logger.error('[/files] Error getting files:', error);+ res.status(400).json({ message: 'Error in request', error: error.message });}-- await fs.unlink(filepath);-};+});router.delete('/', async (req, res) => {try {const { files: _files } = req.body;++ /** @type {MongoFile[]} */const files = _files.filter((file) => {if (!file.file_id) {return false;@@ -47,9 +40,24 @@ router.delete('/', async (req, res) => {}const file_ids = files.map((file) => file.file_id);+ const deletionMethods = {};const promises = [];promises.push(await deleteFiles(file_ids));+for (const file of files) {+ const source = file.source ?? FileSources.local;++ if (deletionMethods[source]) {+ promises.push(deletionMethods[source](req, file));+ continue;+ }++ const { deleteFile } = getStrategyFunctions(source);+ if (!deleteFile) {+ throw new Error(`Delete function not implemented for ${source}`);+ }++ deletionMethods[source] = deleteFile;promises.push(deleteFile(req, file));}diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.jsindex f88b7f2c7a..30d3c3cac6 100644--- a/api/server/routes/files/images.js+++ b/api/server/routes/files/images.js@@ -2,7 +2,7 @@ const { z } = require('zod');const fs = require('fs').promises;const express = require('express');const upload = require('./multer');-const { localStrategy } = require('~/server/services/Files');+const { processImageUpload } = require('~/server/services/Files/process');const { logger } = require('~/config');const router = express.Router();@@ -34,7 +34,8 @@ router.post('/', upload.single('file'), async (req, res) => {uuidSchema.parse(metadata.file_id);metadata.temp_file_id = metadata.file_id;metadata.file_id = req.file_id;- await localStrategy({ req, res, file, metadata });++ await processImageUpload({ req, res, file, metadata });} catch (error) {logger.error('[/files/images] Error processing file:', error);try {diff --git a/api/server/routes/files/index.js b/api/server/routes/files/index.jsindex 74b200c806..9afb900bbe 100644--- a/api/server/routes/files/index.js+++ b/api/server/routes/files/index.js@@ -11,6 +11,7 @@ const {const files = require('./files');const images = require('./images');+const avatar = require('./avatar');router.use(requireJwtAuth);router.use(checkBan);@@ -18,6 +19,6 @@ router.use(uaParser);router.use('/', files);router.use('/images', images);-router.use('/images/avatar', require('./avatar'));+router.use('/images/avatar', avatar);module.exports = router;diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.jsindex ae0158f7d5..d5aea05a37 100644--- a/api/server/routes/files/multer.js+++ b/api/server/routes/files/multer.js@@ -8,7 +8,7 @@ const sizeLimit = 20 * 1024 * 1024; // 20 MBconst storage = multer.diskStorage({destination: function (req, file, cb) {- const outputPath = path.join(req.app.locals.config.imageOutput, 'temp');+ const outputPath = path.join(req.app.locals.paths.imageOutput, 'temp');if (!fs.existsSync(outputPath)) {fs.mkdirSync(outputPath, { recursive: true });}diff --git a/api/server/services/AppService.js b/api/server/services/AppService.jsnew file mode 100644index 0000000000..b1f7cf57d8--- /dev/null+++ b/api/server/services/AppService.js@@ -0,0 +1,27 @@+const { FileSources } = require('librechat-data-provider');+const { initializeFirebase } = require('./Files/Firebase/initialize');+const loadCustomConfig = require('./Config/loadCustomConfig');+const paths = require('~/config/paths');++/**+ *+ * Loads custom config and initializes app-wide variables.+ * @function AppService+ * @param {Express.Application} app - The Express application object.+ */+const AppService = async (app) => {+ const config = (await loadCustomConfig()) ?? {};+ const fileStrategy = config.fileStrategy ?? FileSources.local;+ process.env.CDN_PROVIDER = fileStrategy;++ if (fileStrategy === FileSources.firebase) {+ initializeFirebase();+ }++ app.locals = {+ fileStrategy,+ paths,+ };+};++module.exports = AppService;diff --git a/api/server/services/Endpoints/openAI/buildOptions.js b/api/server/services/Endpoints/openAI/buildOptions.jsindex a1ad232bb7..80037fb4b8 100644--- a/api/server/services/Endpoints/openAI/buildOptions.js+++ b/api/server/services/Endpoints/openAI/buildOptions.js@@ -1,9 +1,11 @@const buildOptions = (endpoint, parsedBody) => {- const { chatGptLabel, promptPrefix, ...rest } = parsedBody;+ const { chatGptLabel, promptPrefix, resendImages, imageDetail, ...rest } = parsedBody;const endpointOption = {endpoint,chatGptLabel,promptPrefix,+ resendImages,+ imageDetail,modelOptions: {...rest,},diff --git a/api/server/services/Files/Firebase/crud.js b/api/server/services/Files/Firebase/crud.jsnew file mode 100644index 0000000000..68f534bcb6--- /dev/null+++ b/api/server/services/Files/Firebase/crud.js@@ -0,0 +1,174 @@+const fetch = require('node-fetch');+const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');+const { getFirebaseStorage } = require('./initialize');++/**+ * Deletes a file from Firebase Storage.+ * @param {string} directory - The directory name+ * @param {string} fileName - The name of the file to delete.+ * @returns {Promise<void>} A promise that resolves when the file is deleted.+ */+async function deleteFile(basePath, fileName) {+ const storage = getFirebaseStorage();+ if (!storage) {+ console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');+ throw new Error('Firebase is not initialized');+ }++ const storageRef = ref(storage, `${basePath}/${fileName}`);++ try {+ await deleteObject(storageRef);+ console.log('File deleted successfully from Firebase Storage');+ } catch (error) {+ console.error('Error deleting file from Firebase Storage:', error.message);+ throw error;+ }+}++/**+ * Saves an file from a given URL to Firebase Storage. The function first initializes the Firebase Storage+ * reference, then uploads the file to a specified basePath in the Firebase Storage. It handles initialization+ * errors and upload errors, logging them to the console. If the upload is successful, the file name is returned.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath+ * in Firebase Storage.+ * @param {string} params.URL - The URL of the file to be uploaded. The file at this URL will be fetched+ * and uploaded to Firebase Storage.+ * @param {string} params.fileName - The name that will be used to save the file in Firebase Storage. This+ * should include the file extension.+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will+ * be stored. Defaults to 'images' if not specified.+ *+ * @returns {Promise<string|null>}+ * A promise that resolves to the file name if the file is successfully uploaded, or null if there+ * is an error in initialization or upload.+ */+async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {+ const storage = getFirebaseStorage();+ if (!storage) {+ console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');+ return null;+ }++ const storageRef = ref(storage, `${basePath}/${userId.toString()}/${fileName}`);++ try {+ await uploadBytes(storageRef, await fetch(URL).then((response) => response.buffer()));+ return fileName;+ } catch (error) {+ console.error('Error uploading file to Firebase Storage:', error.message);+ return null;+ }+}++/**+ * Retrieves the download URL for a specified file from Firebase Storage. This function initializes the+ * Firebase Storage and generates a reference to the file based on the provided basePath and file name. If+ * Firebase Storage is not initialized or if there is an error in fetching the URL, the error is logged+ * to the console.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.fileName - The name of the file for which the URL is to be retrieved. This should+ * include the file extension.+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file is+ * stored. Defaults to 'images' if not specified.+ *+ * @returns {Promise<string|null>}+ * A promise that resolves to the download URL of the file if successful, or null if there is an+ * error in initialization or fetching the URL.+ */+async function getFirebaseURL({ fileName, basePath = 'images' }) {+ const storage = getFirebaseStorage();+ if (!storage) {+ console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');+ return null;+ }++ const storageRef = ref(storage, `${basePath}/${fileName}`);++ try {+ return await getDownloadURL(storageRef);+ } catch (error) {+ console.error('Error fetching file URL from Firebase Storage:', error.message);+ return null;+ }+}++/**+ * Uploads a buffer to Firebase Storage.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath+ * in Firebase Storage.+ * @param {string} params.fileName - The name of the file to be saved in Firebase Storage.+ * @param {string} params.buffer - The buffer to be uploaded.+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will+ * be stored. Defaults to 'images' if not specified.+ *+ * @returns {Promise<string>} - A promise that resolves to the download URL of the uploaded file.+ */+async function saveBufferToFirebase({ userId, buffer, fileName, basePath = 'images' }) {+ const storage = getFirebaseStorage();+ if (!storage) {+ throw new Error('Firebase is not initialized');+ }++ const storageRef = ref(storage, `${basePath}/${userId}/${fileName}`);+ await uploadBytes(storageRef, buffer);++ // Assuming you have a function to get the download URL+ return await getFirebaseURL({ fileName, basePath: `${basePath}/${userId}` });+}++/**+ * Extracts and decodes the file path from a Firebase Storage URL.+ *+ * @param {string} urlString - The Firebase Storage URL.+ * @returns {string} The decoded file path.+ */+function extractFirebaseFilePath(urlString) {+ try {+ const url = new URL(urlString);+ const pathRegex = /\/o\/(.+?)(\?|$)/;+ const match = url.pathname.match(pathRegex);++ if (match && match[1]) {+ return decodeURIComponent(match[1]);+ }++ return '';+ } catch (error) {+ // If URL parsing fails, return an empty string+ return '';+ }+}++/**+ * Deletes a file from Firebase storage. This function determines the filepath from the+ * Firebase storage URL via regex for deletion. Validated by the user's ID.+ *+ * @param {Express.Request} req - The request object from Express.+ * It should contain a `user` object with an `id` property.+ * @param {MongoFile} file - The file object to be deleted.+ *+ * @returns {Promise<void>}+ * A promise that resolves when the file has been successfully deleted from Firebase storage.+ * Throws an error if there is an issue with deletion.+ */+const deleteFirebaseFile = async (req, file) => {+ const fileName = extractFirebaseFilePath(file.filepath);+ if (!fileName.includes(req.user.id)) {+ throw new Error('Invalid file path');+ }+ await deleteFile('', fileName);+};++module.exports = {+ deleteFile,+ getFirebaseURL,+ saveURLToFirebase,+ deleteFirebaseFile,+ saveBufferToFirebase,+};diff --git a/api/server/services/Files/Firebase/images.js b/api/server/services/Files/Firebase/images.jsindex e04902c02f..95b600962f 100644--- a/api/server/services/Files/Firebase/images.js+++ b/api/server/services/Files/Firebase/images.js@@ -1,45 +1,105 @@-const fetch = require('node-fetch');-const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');-const { getFirebaseStorage } = require('./initialize');--async function saveImageToFirebaseStorage(userId, imageUrl, imageName) {- const storage = getFirebaseStorage();- if (!storage) {- console.error('Firebase is not initialized. Cannot save image to Firebase Storage.');- return null;- }+const fs = require('fs');+const path = require('path');+const sharp = require('sharp');+const { resizeImage } = require('../images/resize');+const { saveBufferToFirebase } = require('./crud');+const { updateFile } = require('~/models/File');+const { logger } = require('~/config');- const storageRef = ref(storage, `images/${userId.toString()}/${imageName}`);+/**+ * Converts an image file to the WebP format. The function first resizes the image based on the specified+ * resolution.+ *+ *+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`+ * representing the user, and an `app.locals.paths` object with an `imageOutput` path.+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should+ * have a `path` property that points to the location of the uploaded file.+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.+ *+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}+ * A promise that resolves to an object containing:+ * - filepath: The path where the converted WebP image is saved.+ * - bytes: The size of the converted image in bytes.+ * - width: The width of the converted image.+ * - height: The height of the converted image.+ */+async function uploadImageToFirebase(req, file, resolution = 'high') {+ const inputFilePath = file.path;+ const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);+ const extension = path.extname(inputFilePath);+ const userId = req.user.id;- try {- // Upload image to Firebase Storage using the image URL- await uploadBytes(storageRef, await fetch(imageUrl).then((response) => response.buffer()));- return imageName;- } catch (error) {- console.error('Error uploading image to Firebase Storage:', error.message);- return null;+ let webPBuffer;+ let fileName = path.basename(inputFilePath);+ if (extension.toLowerCase() === '.webp') {+ webPBuffer = resizedBuffer;+ } else {+ webPBuffer = await sharp(resizedBuffer).toFormat('webp').toBuffer();+ // Replace or append the correct extension+ const extRegExp = new RegExp(path.extname(fileName) + '$');+ fileName = fileName.replace(extRegExp, '.webp');+ if (!path.extname(fileName)) {+ fileName += '.webp';+ }}-}-async function getFirebaseStorageImageUrl(imageName) {- const storage = getFirebaseStorage();- if (!storage) {- console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');- return null;- }+ const downloadURL = await saveBufferToFirebase({ userId, buffer: webPBuffer, fileName });++ await fs.promises.unlink(inputFilePath);- const storageRef = ref(storage, `images/${imageName}`);+ const bytes = Buffer.byteLength(webPBuffer);+ return { filepath: downloadURL, bytes, width, height };+}+/**+ * Local: Updates the file and returns the URL in expected order/format+ * for image payload handling: tuple order of [filepath, URL].+ * @param {Object} req - The request object.+ * @param {MongoFile} file - The file object.+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.+ */+async function prepareImageURL(req, file) {+ const { filepath } = file;+ const promises = [];+ promises.push(updateFile({ file_id: file.file_id }));+ promises.push(filepath);+ return await Promise.all(promises);+}++/**+ * Uploads a user's avatar to Firebase Storage and returns the URL.+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.+ *+ * @param {object} params - The parameters object.+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.+ */+async function processFirebaseAvatar({ buffer, User, manual }) {try {- // Get the download URL for the image from Firebase Storage- return `})`;+ const downloadURL = await saveBufferToFirebase({+ userId: User._id.toString(),+ buffer,+ fileName: 'avatar.png',+ });++ const isManual = manual === 'true';++ const url = `${downloadURL}?manual=${isManual}`;++ if (isManual) {+ User.avatar = url;+ await User.save();+ }++ return url;} catch (error) {- console.error('Error fetching image URL from Firebase Storage:', error.message);- return null;+ logger.error('Error uploading profile picture:', error);+ throw error;}}-module.exports = {- saveImageToFirebaseStorage,- getFirebaseStorageImageUrl,-};+module.exports = { uploadImageToFirebase, prepareImageURL, processFirebaseAvatar };diff --git a/api/server/services/Files/Firebase/index.js b/api/server/services/Files/Firebase/index.jsindex 905bf660d4..27ad97a852 100644--- a/api/server/services/Files/Firebase/index.js+++ b/api/server/services/Files/Firebase/index.js@@ -1,7 +1,9 @@+const crud = require('./crud');const images = require('./images');const initialize = require('./initialize');module.exports = {+ ...crud,...images,...initialize,};diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.jsnew file mode 100644index 0000000000..d81c063031--- /dev/null+++ b/api/server/services/Files/Local/crud.js@@ -0,0 +1,174 @@+const fs = require('fs');+const path = require('path');+const axios = require('axios');+const { logger } = require('~/config');+const paths = require('~/config/paths');++/**+ * Saves a file to a specified output path with a new filename.+ *+ * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.+ * @param {string} outputPath - The path where the file should be saved.+ * @param {string} outputFilename - The new filename for the saved file (without extension).+ * @returns {Promise<string>} The full path of the saved file.+ * @throws Will throw an error if the file saving process fails.+ */+async function saveFile(file, outputPath, outputFilename) {+ try {+ if (!fs.existsSync(outputPath)) {+ fs.mkdirSync(outputPath, { recursive: true });+ }++ const fileExtension = path.extname(file.originalname);+ const filenameWithExt = outputFilename + fileExtension;+ const outputFilePath = path.join(outputPath, filenameWithExt);+ fs.copyFileSync(file.path, outputFilePath);+ fs.unlinkSync(file.path);++ return outputFilePath;+ } catch (error) {+ logger.error('[saveFile] Error while saving the file:', error);+ throw error;+ }+}++/**+ * Saves an uploaded image file to a specified directory based on the user's ID and a filename.+ *+ * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.+ * @param {Express.Multer.File} file - The uploaded file object.+ * @param {string} filename - The new filename to assign to the saved image (without extension).+ * @returns {Promise<void>}+ * @throws Will throw an error if the image saving process fails.+ */+const saveLocalImage = async (req, file, filename) => {+ const imagePath = req.app.locals.paths.imageOutput;+ const outputPath = path.join(imagePath, req.user.id ?? '');+ await saveFile(file, outputPath, filename);+};++/**+ * Saves a file from a given URL to a local directory. The function fetches the file using the provided URL,+ * determines the content type, and saves it to a specified local directory with the correct file extension.+ * If the specified directory does not exist, it is created. The function returns the name of the saved file+ * or null in case of an error.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific path+ * in the local file system.+ * @param {string} params.URL - The URL of the file to be downloaded and saved.+ * @param {string} params.fileName - The desired file name for the saved file. This may be modified to include+ * the correct file extension based on the content type.+ * @param {string} [params.basePath='images'] - Optional. The base directory where the file will be saved.+ * Defaults to 'images' if not specified.+ *+ * @returns {Promise<string|null>}+ * A promise that resolves to the file name if the file is successfully saved, or null if there is an error.+ */+async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {+ try {+ // Fetch the file from the URL+ const response = await axios({+ url: URL,+ responseType: 'stream',+ });++ // Get the content type from the response headers+ const contentType = response.headers['content-type'];+ let extension = contentType.split('/').pop();++ // Construct the outputPath based on the basePath and userId+ const outputPath = path.join(paths.publicPath, basePath, userId.toString());++ // Check if the output directory exists, if not, create it+ if (!fs.existsSync(outputPath)) {+ fs.mkdirSync(outputPath, { recursive: true });+ }++ // Replace or append the correct extension+ const extRegExp = new RegExp(path.extname(fileName) + '$');+ fileName = fileName.replace(extRegExp, `.${extension}`);+ if (!path.extname(fileName)) {+ fileName += `.${extension}`;+ }++ // Create a writable stream for the output path+ const outputFilePath = path.join(outputPath, fileName);+ const writer = fs.createWriteStream(outputFilePath);++ // Pipe the response data to the output file+ response.data.pipe(writer);++ return new Promise((resolve, reject) => {+ writer.on('finish', () => resolve(fileName));+ writer.on('error', reject);+ });+ } catch (error) {+ logger.error('[saveFileFromURL] Error while saving the file:', error);+ return null;+ }+}++/**+ * Constructs a local file path for a given file name and base path. This function simply joins the base+ * path and the file name to create a file path. It does not check for the existence of the file at the path.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.fileName - The name of the file for which the path is to be constructed. This should+ * include the file extension.+ * @param {string} [params.basePath='images'] - Optional. The base directory to be used for constructing the file path.+ * Defaults to 'images' if not specified.+ *+ * @returns {string}+ * The constructed local file path.+ */+async function getLocalFileURL({ fileName, basePath = 'images' }) {+ return path.posix.join('/', basePath, fileName);+}++/**+ * Validates if a given filepath is within a specified subdirectory under a base path. This function constructs+ * the expected base path using the base, subfolder, and user id from the request, and then checks if the+ * provided filepath starts with this constructed base path.+ *+ * @param {Express.Request} req - The request object from Express. It should contain a `user` property with an `id`.+ * @param {string} base - The base directory path.+ * @param {string} subfolder - The subdirectory under the base path.+ * @param {string} filepath - The complete file path to be validated.+ *+ * @returns {boolean}+ * Returns true if the filepath is within the specified base and subfolder, false otherwise.+ */+const isValidPath = (req, base, subfolder, filepath) => {+ const normalizedBase = path.resolve(base, subfolder, req.user.id);+ const normalizedFilepath = path.resolve(filepath);+ return normalizedFilepath.startsWith(normalizedBase);+};++/**+ * Deletes a file from the filesystem. This function takes a file object, constructs the full path, and+ * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.+ *+ * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with+ * a `publicPath` property.+ * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is+ * a string representing the path of the file relative to the publicPath.+ *+ * @returns {Promise<void>}+ * A promise that resolves when the file has been successfully deleted, or throws an error if the+ * file path is invalid or if there is an error in deletion.+ */+const deleteLocalFile = async (req, file) => {+ const { publicPath } = req.app.locals.paths;+ const parts = file.filepath.split(path.sep);+ const subfolder = parts[1];+ const filepath = path.join(publicPath, file.filepath);++ if (!isValidPath(req, publicPath, subfolder, filepath)) {+ throw new Error('Invalid file path');+ }++ await fs.promises.unlink(filepath);+};++module.exports = { saveFile, saveLocalImage, saveFileFromURL, getLocalFileURL, deleteLocalFile };diff --git a/api/server/services/Files/Local/images.js b/api/server/services/Files/Local/images.jsnew file mode 100644index 0000000000..63ed5b2f64--- /dev/null+++ b/api/server/services/Files/Local/images.js@@ -0,0 +1,140 @@+const fs = require('fs');+const path = require('path');+const sharp = require('sharp');+const { resizeImage } = require('../images/resize');+const { updateFile } = require('~/models/File');++/**+ * Converts an image file to the WebP format. The function first resizes the image based on the specified+ * resolution.+ *+ * If the original image is already in WebP format, it writes the resized image back. Otherwise,+ * it converts the image to WebP format before saving.+ *+ * The original image is deleted after conversion.+ *+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`+ * representing the user, and an `app.locals.paths` object with an `imageOutput` path.+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should+ * have a `path` property that points to the location of the uploaded file.+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.+ *+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}+ * A promise that resolves to an object containing:+ * - filepath: The path where the converted WebP image is saved.+ * - bytes: The size of the converted image in bytes.+ * - width: The width of the converted image.+ * - height: The height of the converted image.+ */+async function uploadLocalImage(req, file, resolution = 'high') {+ const inputFilePath = file.path;+ const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);+ const extension = path.extname(inputFilePath);++ const { imageOutput } = req.app.locals.paths;+ const userPath = path.join(imageOutput, req.user.id);++ if (!fs.existsSync(userPath)) {+ fs.mkdirSync(userPath, { recursive: true });+ }++ const newPath = path.join(userPath, path.basename(inputFilePath));++ if (extension.toLowerCase() === '.webp') {+ const bytes = Buffer.byteLength(resizedBuffer);+ await fs.promises.writeFile(newPath, resizedBuffer);+ const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));+ return { filepath, bytes, width, height };+ }++ const outputFilePath = newPath.replace(extension, '.webp');+ const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();+ await fs.promises.writeFile(outputFilePath, data);+ const bytes = Buffer.byteLength(data);+ const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));+ await fs.promises.unlink(inputFilePath);+ return { filepath, bytes, width, height };+}++/**+ * Encodes an image file to base64.+ * @param {string} imagePath - The path to the image file.+ * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.+ */+function encodeImage(imagePath) {+ return new Promise((resolve, reject) => {+ fs.readFile(imagePath, (err, data) => {+ if (err) {+ reject(err);+ } else {+ resolve(data.toString('base64'));+ }+ });+ });+}++/**+ * Local: Updates the file and encodes the image to base64,+ * for image payload handling: tuple order of [filepath, base64].+ * @param {Object} req - The request object.+ * @param {MongoFile} file - The file object.+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.+ */+async function prepareImagesLocal(req, file) {+ const { publicPath, imageOutput } = req.app.locals.paths;+ const userPath = path.join(imageOutput, req.user.id);++ if (!fs.existsSync(userPath)) {+ fs.mkdirSync(userPath, { recursive: true });+ }+ const filepath = path.join(publicPath, file.filepath);++ const promises = [];+ promises.push(updateFile({ file_id: file.file_id }));+ promises.push(encodeImage(filepath));+ return await Promise.all(promises);+}++/**+ * Uploads a user's avatar to Firebase Storage and returns the URL.+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.+ *+ * @param {object} params - The parameters object.+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.+ */+async function processLocalAvatar({ buffer, User, manual }) {+ const userDir = path.resolve(+ __dirname,+ '..',+ '..',+ '..',+ '..',+ '..',+ 'client',+ 'public',+ 'images',+ User._id.toString(),+ );+ const fileName = `avatar-${new Date().getTime()}.png`;+ const urlRoute = `/images/${User._id.toString()}/${fileName}`;+ const avatarPath = path.join(userDir, fileName);++ await fs.promises.mkdir(userDir, { recursive: true });+ await fs.promises.writeFile(avatarPath, buffer);++ const isManual = manual === 'true';+ let url = `${urlRoute}?manual=${isManual}`;++ if (isManual) {+ User.avatar = url;+ await User.save();+ }++ return url;+}++module.exports = { uploadLocalImage, encodeImage, prepareImagesLocal, processLocalAvatar };diff --git a/api/server/services/Files/Local/index.js b/api/server/services/Files/Local/index.jsnew file mode 100644index 0000000000..cb44238bcc--- /dev/null+++ b/api/server/services/Files/Local/index.js@@ -0,0 +1,7 @@+const images = require('./images');+const crud = require('./crud');++module.exports = {+ ...crud,+ ...images,+};diff --git a/api/server/services/Files/images/avatar.js b/api/server/services/Files/images/avatar.jsnew file mode 100644index 0000000000..490fc86171--- /dev/null+++ b/api/server/services/Files/images/avatar.js@@ -0,0 +1,78 @@+const sharp = require('sharp');+const fs = require('fs').promises;+const fetch = require('node-fetch');+const User = require('~/models/User');+const { getStrategyFunctions } = require('~/server/services/Files/strategies');+const { logger } = require('~/config');++async function convertToWebP(inputBuffer) {+ return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();+}++/**+ * Uploads an avatar image for a user. This function can handle various types of input (URL, Buffer, or File object),+ * processes the image to a square format, converts it to WebP format, and then uses a specified file strategy for+ * further processing. It performs validation on the user ID and the input type. The function can throw errors for+ * invalid input types, fetching issues, or other processing errors.+ *+ * @param {Object} params - The parameters object.+ * @param {string} params.userId - The unique identifier of the user for whom the avatar is being uploaded.+ * @param {FileSources} params.fileStrategy - The file handling strategy to use, determining how the avatar is processed.+ * @param {(string|Buffer|File)} params.input - The input representing the avatar image. Can be a URL (string),+ * a Buffer, or a File object.+ * @param {string} params.manual - A string flag indicating whether the upload process is manual.+ *+ * @returns {Promise<any>}+ * A promise that resolves to the result of the `processAvatar` function, specific to the chosen file+ * strategy. Throws an error if any step in the process fails.+ *+ * @throws {Error} Throws an error if the user ID is undefined, the input type is invalid, the image fetching fails,+ * or any other error occurs during the processing.+ */+async function uploadAvatar({ userId, fileStrategy, input, manual }) {+ try {+ if (userId === undefined) {+ throw new Error('User ID is undefined');+ }+ const _id = userId;+ // TODO: remove direct use of Model, `User`+ const oldUser = await User.findOne({ _id });++ let imageBuffer;+ if (typeof input === 'string') {+ const response = await fetch(input);++ if (!response.ok) {+ throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);+ }+ imageBuffer = await response.buffer();+ } else if (input instanceof Buffer) {+ imageBuffer = input;+ } else if (typeof input === 'object' && input instanceof File) {+ const fileContent = await fs.readFile(input.path);+ imageBuffer = Buffer.from(fileContent);+ } else {+ throw new Error('Invalid input type. Expected URL, Buffer, or File.');+ }++ const { width, height } = await sharp(imageBuffer).metadata();+ const minSize = Math.min(width, height);+ const squaredBuffer = await sharp(imageBuffer)+ .extract({+ left: Math.floor((width - minSize) / 2),+ top: Math.floor((height - minSize) / 2),+ width: minSize,+ height: minSize,+ })+ .toBuffer();++ const webPBuffer = await convertToWebP(squaredBuffer);+ const { processAvatar } = getStrategyFunctions(fileStrategy);+ return await processAvatar({ buffer: webPBuffer, User: oldUser, manual });+ } catch (error) {+ logger.error('Error uploading the avatar:', error);+ throw error;+ }+}++module.exports = uploadAvatar;diff --git a/api/server/services/Files/images/avatar/firebaseStrategy.js b/api/server/services/Files/images/avatar/firebaseStrategy.jsdeleted file mode 100644index 9c000b43ec..0000000000--- a/api/server/services/Files/images/avatar/firebaseStrategy.js+++ /dev/null@@ -1,29 +0,0 @@-const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');-const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');-const { logger } = require('~/config');--async function firebaseStrategy(userId, webPBuffer, oldUser, manual) {- try {- const storage = getFirebaseStorage();- if (!storage) {- throw new Error('Firebase is not initialized.');- }- const avatarRef = ref(storage, `images/${userId.toString()}/avatar`);-- await uploadBytes(avatarRef, webPBuffer);- const urlFirebase = await getDownloadURL(avatarRef);- const isManual = manual === 'true';-- const url = `${urlFirebase}?manual=${isManual}`;- if (isManual) {- oldUser.avatar = url;- await oldUser.save();- }- return url;- } catch (error) {- logger.error('Error uploading profile picture:', error);- throw error;- }-}--module.exports = firebaseStrategy;diff --git a/api/server/services/Files/images/avatar/localStrategy.js b/api/server/services/Files/images/avatar/localStrategy.jsdeleted file mode 100644index 021beda7d1..0000000000--- a/api/server/services/Files/images/avatar/localStrategy.js+++ /dev/null@@ -1,32 +0,0 @@-const fs = require('fs').promises;-const path = require('path');--async function localStrategy(userId, webPBuffer, oldUser, manual) {- const userDir = path.resolve(- __dirname,- '..',- '..',- '..',- '..',- '..',- '..',- 'client',- 'public',- 'images',- userId,- );- let avatarPath = path.join(userDir, 'avatar.png');- const urlRoute = `/images/${userId}/avatar.png`;- await fs.mkdir(userDir, { recursive: true });- await fs.writeFile(avatarPath, webPBuffer);- const isManual = manual === 'true';- let url = `${urlRoute}?manual=${isManual}×tamp=${new Date().getTime()}`;- if (isManual) {- oldUser.avatar = url;- await oldUser.save();- }-- return url;-}--module.exports = localStrategy;diff --git a/api/server/services/Files/images/avatar/uploadAvatar.js b/api/server/services/Files/images/avatar/uploadAvatar.jsdeleted file mode 100644index 0726df9a4d..0000000000--- a/api/server/services/Files/images/avatar/uploadAvatar.js+++ /dev/null@@ -1,63 +0,0 @@-const sharp = require('sharp');-const fetch = require('node-fetch');-const fs = require('fs').promises;-const User = require('~/models/User');-const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');-const firebaseStrategy = require('./firebaseStrategy');-const localStrategy = require('./localStrategy');-const { logger } = require('~/config');--async function convertToWebP(inputBuffer) {- return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();-}--async function uploadAvatar(userId, input, manual) {- try {- if (userId === undefined) {- throw new Error('User ID is undefined');- }- const _id = userId;- // TODO: remove direct use of Model, `User`- const oldUser = await User.findOne({ _id });- let imageBuffer;- if (typeof input === 'string') {- const response = await fetch(input);-- if (!response.ok) {- throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);- }- imageBuffer = await response.buffer();- } else if (input instanceof Buffer) {- imageBuffer = input;- } else if (typeof input === 'object' && input instanceof File) {- const fileContent = await fs.readFile(input.path);- imageBuffer = Buffer.from(fileContent);- } else {- throw new Error('Invalid input type. Expected URL, Buffer, or File.');- }- const { width, height } = await sharp(imageBuffer).metadata();- const minSize = Math.min(width, height);- const squaredBuffer = await sharp(imageBuffer)- .extract({- left: Math.floor((width - minSize) / 2),- top: Math.floor((height - minSize) / 2),- width: minSize,- height: minSize,- })- .toBuffer();- const webPBuffer = await convertToWebP(squaredBuffer);- const storage = getFirebaseStorage();- if (storage) {- const url = await firebaseStrategy(userId, webPBuffer, oldUser, manual);- return url;- }-- const url = await localStrategy(userId, webPBuffer, oldUser, manual);- return url;- } catch (error) {- logger.error('Error uploading the avatar:', error);- throw error;- }-}--module.exports = uploadAvatar;diff --git a/api/server/services/Files/images/convert.js b/api/server/services/Files/images/convert.jsdeleted file mode 100644index 2de0fd2177..0000000000--- a/api/server/services/Files/images/convert.js+++ /dev/null@@ -1,36 +0,0 @@-const path = require('path');-const sharp = require('sharp');-const fs = require('fs');-const { resizeImage } = require('./resize');--async function convertToWebP(req, file, resolution = 'high') {- const inputFilePath = file.path;- const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);- const extension = path.extname(inputFilePath);-- const { imageOutput } = req.app.locals.config;- const userPath = path.join(imageOutput, req.user.id);-- if (!fs.existsSync(userPath)) {- fs.mkdirSync(userPath, { recursive: true });- }-- const newPath = path.join(userPath, path.basename(inputFilePath));-- if (extension.toLowerCase() === '.webp') {- const bytes = Buffer.byteLength(resizedBuffer);- await fs.promises.writeFile(newPath, resizedBuffer);- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));- return { filepath, bytes, width, height };- }-- const outputFilePath = newPath.replace(extension, '.webp');- const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();- await fs.promises.writeFile(outputFilePath, data);- const bytes = Buffer.byteLength(data);- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));- await fs.promises.unlink(inputFilePath);- return { filepath, bytes, width, height };-}--module.exports = { convertToWebP };diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.jsindex 30428ffabc..809ec0e840 100644--- a/api/server/services/Files/images/encode.js+++ b/api/server/services/Files/images/encode.js@@ -1,45 +1,5 @@-const fs = require('fs');-const path = require('path');-const { EModelEndpoint } = require('librechat-data-provider');-const { updateFile } = require('~/models');--/**- * Encodes an image file to base64.- * @param {string} imagePath - The path to the image file.- * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.- */-function encodeImage(imagePath) {- return new Promise((resolve, reject) => {- fs.readFile(imagePath, (err, data) => {- if (err) {- reject(err);- } else {- resolve(data.toString('base64'));- }- });- });-}--/**- * Updates the file and encodes the image.- * @param {Object} req - The request object.- * @param {Object} file - The file object.- * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.- */-async function updateAndEncode(req, file) {- const { publicPath, imageOutput } = req.app.locals.config;- const userPath = path.join(imageOutput, req.user.id);-- if (!fs.existsSync(userPath)) {- fs.mkdirSync(userPath, { recursive: true });- }- const filepath = path.join(publicPath, file.filepath);-- const promises = [];- promises.push(updateFile({ file_id: file.file_id }));- promises.push(encodeImage(filepath));- return await Promise.all(promises);-}+const { EModelEndpoint, FileSources } = require('librechat-data-provider');+const { getStrategyFunctions } = require('../strategies');/*** Encodes and formats the given files.@@ -50,25 +10,42 @@ async function updateAndEncode(req, file) {*/async function encodeAndFormat(req, files, endpoint) {const promises = [];+ const encodingMethods = {};+for (let file of files) {- promises.push(updateAndEncode(req, file));+ const source = file.source ?? FileSources.local;++ if (encodingMethods[source]) {+ promises.push(encodingMethods[source](req, file));+ continue;+ }++ const { prepareImagePayload } = getStrategyFunctions(source);+ if (!prepareImagePayload) {+ throw new Error(`Encoding function not implemented for ${source}`);+ }++ encodingMethods[source] = prepareImagePayload;+ promises.push(prepareImagePayload(req, file));}- // TODO: make detail configurable, as of now resizing is done- // to prefer "high" but "low" may be used if the image is small enough- const detail = req.body.detail ?? 'auto';- const encodedImages = await Promise.all(promises);+ const detail = req.body.imageDetail ?? 'auto';++ /** @type {Array<[MongoFile, string]>} */+ const formattedImages = await Promise.all(promises);const result = {files: [],image_urls: [],};- for (const [file, base64] of encodedImages) {+ for (const [file, imageContent] of formattedImages) {const imagePart = {type: 'image_url',image_url: {- url: `data:image/webp;base64,${base64}`,+ url: imageContent.startsWith('http')+ ? imageContent+ : `data:image/webp;base64,${imageContent}`,detail,},};@@ -81,17 +58,16 @@ async function encodeAndFormat(req, files, endpoint) {result.files.push({file_id: file.file_id,- filepath: file.filepath,- filename: file.filename,- type: file.type,- height: file.height,- width: file.width,+ // filepath: file.filepath,+ // filename: file.filename,+ // type: file.type,+ // height: file.height,+ // width: file.width,});}return result;}module.exports = {- encodeImage,encodeAndFormat,};diff --git a/api/server/services/Files/images/index.js b/api/server/services/Files/images/index.jsindex fa49eb9535..1438887e6d 100644--- a/api/server/services/Files/images/index.js+++ b/api/server/services/Files/images/index.js@@ -1,15 +1,13 @@-const convert = require('./convert');+const avatar = require('./avatar');const encode = require('./encode');const parse = require('./parse');const resize = require('./resize');const validate = require('./validate');-const uploadAvatar = require('./avatar/uploadAvatar');module.exports = {- ...convert,...encode,...parse,...resize,...validate,- uploadAvatar,+ avatar,};diff --git a/api/server/services/Files/images/parse.js b/api/server/services/Files/images/parse.jsindex 5a1113c97e..1b0f7e4738 100644--- a/api/server/services/Files/images/parse.js+++ b/api/server/services/Files/images/parse.js@@ -1,7 +1,7 @@const URL = require('url').URL;const path = require('path');-const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;+const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg|webp)$/i;/*** Extracts the image basename from a given URL.@@ -22,6 +22,24 @@ function getImageBasename(urlString) {}}+/**+ * Extracts the basename of a file from a given URL.+ *+ * @param {string} urlString - The URL string from which the file basename is to be extracted.+ * @returns {string} The basename of the file from the URL.+ * Returns an empty string if the URL parsing fails.+ */+function getFileBasename(urlString) {+ try {+ const url = new URL(urlString);+ return path.basename(url.pathname);+ } catch (error) {+ // If URL parsing fails, return an empty string+ return '';+ }+}+module.exports = {getImageBasename,+ getFileBasename,};diff --git a/api/server/services/Files/index.js b/api/server/services/Files/index.jsdeleted file mode 100644index 47d47690cb..0000000000--- a/api/server/services/Files/index.js+++ /dev/null@@ -1,9 +0,0 @@-const localStrategy = require('./localStrategy');-const process = require('./process');-const save = require('./save');--module.exports = {- ...save,- ...process,- localStrategy,-};diff --git a/api/server/services/Files/localStrategy.js b/api/server/services/Files/localStrategy.jsdeleted file mode 100644index 0b711c7763..0000000000--- a/api/server/services/Files/localStrategy.js+++ /dev/null@@ -1,36 +0,0 @@-const { createFile } = require('~/models');-const { convertToWebP } = require('./images/convert');--/**- * Applies the local strategy for image uploads.- * Saves file metadata to the database with an expiry TTL.- * Files must be deleted from the server filesystem manually.- *- * @param {Object} params - The parameters object.- * @param {Express.Request} params.req - The Express request object.- * @param {Express.Response} params.res - The Express response object.- * @param {Express.Multer.File} params.file - The uploaded file.- * @param {ImageMetadata} params.metadata - Additional metadata for the file.- * @returns {Promise<void>}- */-const localStrategy = async ({ req, res, file, metadata }) => {- const { file_id, temp_file_id } = metadata;- const { filepath, bytes, width, height } = await convertToWebP(req, file);- const result = await createFile(- {- user: req.user.id,- file_id,- temp_file_id,- bytes,- filepath,- filename: file.originalname,- type: 'image/webp',- width,- height,- },- true,- );- res.status(200).json({ message: 'File uploaded and processed successfully', ...result });-};--module.exports = localStrategy;diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.jsindex c6ab3ca139..4ee9510b4f 100644--- a/api/server/services/Files/process.js+++ b/api/server/services/Files/process.js@@ -1,17 +1,6 @@-const { updateFileUsage } = require('~/models');--// const mapImageUrls = (files, detail) => {-// return files-// .filter((file) => file.type.includes('image'))-// .map((file) => ({-// type: 'image_url',-// image_url: {-// /* Temporarily set to path to encode later */-// url: file.filepath,-// detail,-// },-// }));-// };+const { updateFileUsage, createFile } = require('~/models');+const { getStrategyFunctions } = require('./strategies');+const { logger } = require('~/config');const processFiles = async (files) => {const promises = [];@@ -24,6 +13,76 @@ const processFiles = async (files) => {return await Promise.all(promises);};+/**+ * Processes a file URL using a specified file handling strategy. This function accepts a strategy name,+ * fetches the corresponding file processing functions (for saving and retrieving file URLs), and then+ * executes these functions in sequence. It first saves the file using the provided URL and then retrieves+ * the URL of the saved file. If any error occurs during this process, it logs the error and throws an+ * exception with an appropriate message.+ *+ * @param {Object} params - The parameters object.+ * @param {FileSources} params.fileStrategy - The file handling strategy to use. Must be a value from the+ * `FileSources` enum, which defines different file handling+ * strategies (like saving to Firebase, local storage, etc.).+ * @param {string} params.userId - The user's unique identifier. Used for creating user-specific paths or+ * references in the file handling process.+ * @param {string} params.URL - The URL of the file to be processed.+ * @param {string} params.fileName - The name that will be used to save the file. This should include the+ * file extension.+ * @param {string} params.basePath - The base path or directory where the file will be saved or retrieved from.+ *+ * @returns {Promise<string>}+ * A promise that resolves to the URL of the processed file. It throws an error if the file processing+ * fails at any stage.+ */+const processFileURL = async ({ fileStrategy, userId, URL, fileName, basePath }) => {+ const { saveURL, getFileURL } = getStrategyFunctions(fileStrategy);+ try {+ await saveURL({ userId, URL, fileName, basePath });+ return await getFileURL({ fileName: `${userId}/${fileName}`, basePath });+ } catch (error) {+ logger.error(`Error while processing the image with ${fileStrategy}:`, error);+ throw new Error(`Failed to process the image with ${fileStrategy}. ${error.message}`);+ }+};++/**+ * Applies the current strategy for image uploads.+ * Saves file metadata to the database with an expiry TTL.+ * Files must be deleted from the server filesystem manually.+ *+ * @param {Object} params - The parameters object.+ * @param {Express.Request} params.req - The Express request object.+ * @param {Express.Response} params.res - The Express response object.+ * @param {Express.Multer.File} params.file - The uploaded file.+ * @param {ImageMetadata} params.metadata - Additional metadata for the file.+ * @returns {Promise<void>}+ */+const processImageUpload = async ({ req, res, file, metadata }) => {+ const source = req.app.locals.fileStrategy;+ const { handleImageUpload } = getStrategyFunctions(source);+ const { file_id, temp_file_id } = metadata;+ const { filepath, bytes, width, height } = await handleImageUpload(req, file);+ const result = await createFile(+ {+ user: req.user.id,+ file_id,+ temp_file_id,+ bytes,+ filepath,+ filename: file.originalname,+ source,+ type: 'image/webp',+ width,+ height,+ },+ true,+ );+ res.status(200).json({ message: 'File uploaded and processed successfully', ...result });+};+module.exports = {+ processImageUpload,processFiles,+ processFileURL,};diff --git a/api/server/services/Files/save.js b/api/server/services/Files/save.jsdeleted file mode 100644index 08f6a0d5cc..0000000000--- a/api/server/services/Files/save.js+++ /dev/null@@ -1,48 +0,0 @@-const fs = require('fs');-const path = require('path');-const { logger } = require('~/config');--/**- * Saves a file to a specified output path with a new filename.- *- * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.- * @param {string} outputPath - The path where the file should be saved.- * @param {string} outputFilename - The new filename for the saved file (without extension).- * @returns {Promise<string>} The full path of the saved file.- * @throws Will throw an error if the file saving process fails.- */-async function saveFile(file, outputPath, outputFilename) {- try {- if (!fs.existsSync(outputPath)) {- fs.mkdirSync(outputPath, { recursive: true });- }-- const fileExtension = path.extname(file.originalname);- const filenameWithExt = outputFilename + fileExtension;- const outputFilePath = path.join(outputPath, filenameWithExt);- fs.copyFileSync(file.path, outputFilePath);- fs.unlinkSync(file.path);-- return outputFilePath;- } catch (error) {- logger.error('[saveFile] Error while saving the file:', error);- throw error;- }-}--/**- * Saves an uploaded image file to a specified directory based on the user's ID and a filename.- *- * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.- * @param {Express.Multer.File} file - The uploaded file object.- * @param {string} filename - The new filename to assign to the saved image (without extension).- * @returns {Promise<void>}- * @throws Will throw an error if the image saving process fails.- */-const saveLocalImage = async (req, file, filename) => {- const imagePath = req.app.locals.config.imageOutput;- const outputPath = path.join(imagePath, req.user.id ?? '');- await saveFile(file, outputPath, filename);-};--module.exports = { saveFile, saveLocalImage };diff --git a/api/server/services/Files/strategies.js b/api/server/services/Files/strategies.jsnew file mode 100644index 0000000000..4e20186043--- /dev/null+++ b/api/server/services/Files/strategies.js@@ -0,0 +1,54 @@+const { FileSources } = require('librechat-data-provider');+const {+ getFirebaseURL,+ prepareImageURL,+ saveURLToFirebase,+ deleteFirebaseFile,+ uploadImageToFirebase,+ processFirebaseAvatar,+} = require('./Firebase');+const {+ getLocalFileURL,+ saveFileFromURL,+ deleteLocalFile,+ uploadLocalImage,+ prepareImagesLocal,+ processLocalAvatar,+} = require('./Local');++// Firebase Strategy Functions+const firebaseStrategy = () => ({+ // saveFile:+ saveURL: saveURLToFirebase,+ getFileURL: getFirebaseURL,+ deleteFile: deleteFirebaseFile,+ prepareImagePayload: prepareImageURL,+ processAvatar: processFirebaseAvatar,+ handleImageUpload: uploadImageToFirebase,+});++// Local Strategy Functions+const localStrategy = () => ({+ // saveFile: ,+ saveURL: saveFileFromURL,+ getFileURL: getLocalFileURL,+ deleteFile: deleteLocalFile,+ processAvatar: processLocalAvatar,+ handleImageUpload: uploadLocalImage,+ prepareImagePayload: prepareImagesLocal,+});++// Strategy Selector+const getStrategyFunctions = (fileSource) => {+ if (fileSource === FileSources.firebase) {+ return firebaseStrategy();+ } else if (fileSource === FileSources.local) {+ return localStrategy();+ } else {+ throw new Error('Invalid file source');+ }+};++module.exports = {+ getStrategyFunctions,+};diff --git a/api/server/socialLogins.js b/api/server/socialLogins.jsindex af61db73e9..4abe278b84 100644--- a/api/server/socialLogins.js+++ b/api/server/socialLogins.js@@ -10,6 +10,10 @@ const {} = require('../strategies');const client = require('../cache/redis');+/**+ *+ * @param {Express.Application} app+ */const configureSocialLogins = (app) => {if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) {passport.use(googleLogin());diff --git a/api/strategies/discordStrategy.js b/api/strategies/discordStrategy.jsindex 994554200c..f7b042f940 100644--- a/api/strategies/discordStrategy.js+++ b/api/strategies/discordStrategy.js@@ -1,12 +1,14 @@const { Strategy: DiscordStrategy } = require('passport-discord');+const { createNewUser, handleExistingUser } = require('./process');const { logger } = require('~/config');const User = require('~/models/User');-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');const discordLogin = async (accessToken, refreshToken, profile, cb) => {try {const email = profile.email;const discordId = profile.id;++ // TODO: remove direct access of User modelconst oldUser = await User.findOne({ email });const ALLOW_SOCIAL_REGISTRATION =process.env.ALLOW_SOCIAL_REGISTRATION?.toLowerCase() === 'true';@@ -21,12 +23,20 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {}if (oldUser) {- await handleExistingUser(oldUser, avatarUrl, useFirebase);+ await handleExistingUser(oldUser, avatarUrl);return cb(null, oldUser);}if (ALLOW_SOCIAL_REGISTRATION) {- const newUser = await createNewUser(profile, discordId, email, avatarUrl, useFirebase);+ const newUser = await createNewUser({+ email,+ avatarUrl,+ provider: 'discord',+ providerKey: 'discordId',+ providerId: discordId,+ username: profile.username,+ name: profile.global_name,+ });return cb(null, newUser);}} catch (err) {@@ -35,38 +45,6 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {}};-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {- oldUser.avatar = avatarUrl;- await oldUser.save();- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {- const userId = oldUser._id;- const newavatarUrl = await uploadAvatar(userId, avatarUrl);- oldUser.avatar = newavatarUrl;- await oldUser.save();- }-};--const createNewUser = async (profile, discordId, email, avatarUrl, useFirebase) => {- const newUser = await new User({- provider: 'discord',- discordId,- username: profile.username,- email,- name: profile.global_name,- avatar: avatarUrl,- }).save();-- if (useFirebase) {- const userId = newUser._id;- const newavatarUrl = await uploadAvatar(userId, avatarUrl);- newUser.avatar = newavatarUrl;- await newUser.save();- }-- return newUser;-};-module.exports = () =>new DiscordStrategy({diff --git a/api/strategies/facebookStrategy.js b/api/strategies/facebookStrategy.jsindex b8915b2cc4..dcf7beb797 100644--- a/api/strategies/facebookStrategy.js+++ b/api/strategies/facebookStrategy.js@@ -1,7 +1,7 @@const FacebookStrategy = require('passport-facebook').Strategy;+const { createNewUser, handleExistingUser } = require('./process');const { logger } = require('~/config');const User = require('~/models/User');-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');const facebookLogin = async (accessToken, refreshToken, profile, cb) => {try {@@ -13,12 +13,20 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {const avatarUrl = profile.photos[0]?.value;if (oldUser) {- await handleExistingUser(oldUser, avatarUrl, useFirebase);+ await handleExistingUser(oldUser, avatarUrl);return cb(null, oldUser);}if (ALLOW_SOCIAL_REGISTRATION) {- const newUser = await createNewUser(profile, facebookId, email, avatarUrl, useFirebase);+ const newUser = await createNewUser({+ email,+ avatarUrl,+ provider: 'facebook',+ providerKey: 'facebookId',+ providerId: facebookId,+ username: profile.displayName,+ name: profile.name?.givenName + ' ' + profile.name?.familyName,+ });return cb(null, newUser);}} catch (err) {@@ -27,38 +35,6 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {}};-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {- oldUser.avatar = avatarUrl;- await oldUser.save();- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {- const userId = oldUser._id;- const newavatarUrl = await uploadAvatar(userId, avatarUrl);- oldUser.avatar = newavatarUrl;- await oldUser.save();- }-};--const createNewUser = async (profile, facebookId, email, avatarUrl, useFirebase) => {- const newUser = await new User({- provider: 'facebook',- facebookId,- username: profile.displayName,- email,- name: profile.name?.givenName + ' ' + profile.name?.familyName,- avatar: avatarUrl,- }).save();-- if (useFirebase) {- const userId = newUser._id;- const newavatarUrl = await uploadAvatar(userId, avatarUrl);- newUser.avatar = newavatarUrl;- await newUser.save();- }-- return newUser;-};-module.exports = () =>new FacebookStrategy({diff --git a/api/strategies/githubStrategy.js b/api/strategies/githubStrategy.jsindex c8480d50c1..065568f92f 100644--- a/api/strategies/githubStrategy.js+++ b/api/strategies/githubStrategy.js@@ -1,7 +1,7 @@const { Strategy: GitHubStrategy } = require('passport-github2');+const { createNewUser, handleExistingUser } = require('./process');const { logger } = require('~/config');const User = require('~/models/User');-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');const githubLogin = async (accessToken, refreshToken, profile, cb) => {try {@@ -13,12 +13,21 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {const avatarUrl = profile.photos[0].value;if (oldUser) {- await handleExistingUser(oldUser, avatarUrl, useFirebase);+ await handleExistingUser(oldUser, avatarUrl);return cb(null, oldUser);}if (ALLOW_SOCIAL_REGISTRATION) {- const newUser = await createNewUser(profile, githubId, email, avatarUrl, useFirebase);+ const newUser = await createNewUser({+ email,+ avatarUrl,+ provider: 'github',+ providerKey: 'githubId',+ providerId: githubId,+ username: profile.username,+ name: profile.displayName,+ emailVerified: profile.emails[0].verified,+ });return cb(null, newUser);}} catch (err) {@@ -27,39 +36,6 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {}};-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {- oldUser.avatar = avatarUrl;- await oldUser.save();- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {- const userId = oldUser._id;- const avatarURL = await uploadAvatar(userId, avatarUrl);- oldUser.avatar = avatarURL;- await oldUser.save();- }-};--const createNewUser = async (profile, githubId, email, avatarUrl, useFirebase) => {- const newUser = await new User({- provider: 'github',- githubId,- username: profile.username,- email,- emailVerified: profile.emails[0].verified,- name: profile.displayName,- avatar: avatarUrl,- }).save();-- if (useFirebase) {- const userId = newUser._id;- const avatarURL = await uploadAvatar(userId, avatarUrl);- newUser.avatar = avatarURL;- await newUser.save();- }-- return newUser;-};-module.exports = () =>new GitHubStrategy({diff --git a/api/strategies/googleStrategy.js b/api/strategies/googleStrategy.jsindex d013cc8e8f..0eff48ee09 100644--- a/api/strategies/googleStrategy.js+++ b/api/strategies/googleStrategy.js@@ -1,7 +1,7 @@const { Strategy: GoogleStrategy } = require('passport-google-oauth20');+const { createNewUser, handleExistingUser } = require('./process');const { logger } = require('~/config');const User = require('~/models/User');-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');const googleLogin = async (accessToken, refreshToken, profile, cb) => {try {@@ -13,12 +13,21 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {const avatarUrl = profile.photos[0].value;if (oldUser) {- await handleExistingUser(oldUser, avatarUrl, useFirebase);+ await handleExistingUser(oldUser, avatarUrl);return cb(null, oldUser);}if (ALLOW_SOCIAL_REGISTRATION) {- const newUser = await createNewUser(profile, googleId, email, avatarUrl, useFirebase);+ const newUser = await createNewUser({+ email,+ avatarUrl,+ provider: 'google',+ providerKey: 'googleId',+ providerId: googleId,+ username: profile.name.givenName,+ name: `${profile.name.givenName} ${profile.name.familyName}`,+ emailVerified: profile.emails[0].verified,+ });return cb(null, newUser);}} catch (err) {@@ -27,39 +36,6 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {}};-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {- if ((!useFirebase && !oldUser.avatar.includes('?manual=true')) || oldUser.avatar === null) {- oldUser.avatar = avatarUrl;- await oldUser.save();- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {- const userId = oldUser._id;- const avatarURL = await uploadAvatar(userId, avatarUrl);- oldUser.avatar = avatarURL;- await oldUser.save();- }-};--const createNewUser = async (profile, googleId, email, avatarUrl, useFirebase) => {- const newUser = await new User({- provider: 'google',- googleId,- username: profile.name.givenName,- email,- emailVerified: profile.emails[0].verified,- name: `${profile.name.givenName} ${profile.name.familyName}`,- avatar: avatarUrl,- }).save();-- if (useFirebase) {- const userId = newUser._id;- const avatarURL = await uploadAvatar(userId, avatarUrl);- newUser.avatar = avatarURL;- await newUser.save();- }-- return newUser;-};-module.exports = () =>new GoogleStrategy({diff --git a/api/strategies/process.js b/api/strategies/process.jsnew file mode 100644index 0000000000..f5a12a26a2--- /dev/null+++ b/api/strategies/process.js@@ -0,0 +1,92 @@+const { FileSources } = require('librechat-data-provider');+const uploadAvatar = require('~/server/services/Files/images/avatar');+const User = require('~/models/User');++/**+ * Updates the avatar URL of an existing user. If the user's avatar URL does not include the query parameter+ * '?manual=true', it updates the user's avatar with the provided URL. For local file storage, it directly updates+ * the avatar URL, while for other storage types, it processes the avatar URL using the specified file strategy.+ *+ * @param {User} oldUser - The existing user object that needs to be updated. Expected to have an 'avatar' property.+ * @param {string} avatarUrl - The new avatar URL to be set for the user.+ *+ * @returns {Promise<void>}+ * The function updates the user's avatar and saves the user object. It does not return any value.+ *+ * @throws {Error} Throws an error if there's an issue saving the updated user object.+ */+const handleExistingUser = async (oldUser, avatarUrl) => {+ const fileStrategy = process.env.CDN_PROVIDER;+ const isLocal = fileStrategy === FileSources.local;++ if (isLocal && !oldUser.avatar.includes('?manual=true')) {+ oldUser.avatar = avatarUrl;+ await oldUser.save();+ } else if (!isLocal && !oldUser.avatar.includes('?manual=true')) {+ const userId = oldUser._id;+ const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });+ oldUser.avatar = newavatarUrl;+ await oldUser.save();+ }+};++/**+ * Creates a new user with the provided user details. If the file strategy is not local, the avatar URL is+ * processed using the specified file strategy. The new user is saved to the database with the processed or+ * original avatar URL.+ *+ * @param {Object} params - The parameters object for user creation.+ * @param {string} params.email - The email of the new user.+ * @param {string} params.avatarUrl - The avatar URL of the new user.+ * @param {string} params.provider - The provider of the user's account.+ * @param {string} params.providerKey - The key to identify the provider in the user model.+ * @param {string} params.providerId - The provider-specific ID of the user.+ * @param {string} params.username - The username of the new user.+ * @param {string} params.name - The name of the new user.+ * @param {boolean} [params.emailVerified=false] - Optional. Indicates whether the user's email is verified. Defaults to false.+ *+ * @returns {Promise<User>}+ * A promise that resolves to the newly created user object.+ *+ * @throws {Error} Throws an error if there's an issue creating or saving the new user object.+ */+const createNewUser = async ({+ email,+ avatarUrl,+ provider,+ providerKey,+ providerId,+ username,+ name,+ emailVerified,+}) => {+ const update = {+ email,+ avatar: avatarUrl,+ provider,+ [providerKey]: providerId,+ username,+ name,+ emailVerified,+ };++ // TODO: remove direct access of User model+ const newUser = await new User(update).save();++ const fileStrategy = process.env.CDN_PROVIDER;+ const isLocal = fileStrategy === FileSources.local;++ if (!isLocal) {+ const userId = newUser._id;+ const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });+ newUser.avatar = newavatarUrl;+ await newUser.save();+ }++ return newUser;+};++module.exports = {+ handleExistingUser,+ createNewUser,+};diff --git a/api/test/__mocks__/logger.js b/api/test/__mocks__/logger.jsindex 455ada0de0..caeb004e39 100644--- a/api/test/__mocks__/logger.js+++ b/api/test/__mocks__/logger.js@@ -1,3 +1,43 @@+jest.mock('winston', () => {+ const mockFormatFunction = jest.fn((fn) => fn);++ mockFormatFunction.colorize = jest.fn();+ mockFormatFunction.combine = jest.fn();+ mockFormatFunction.label = jest.fn();+ mockFormatFunction.timestamp = jest.fn();+ mockFormatFunction.printf = jest.fn();+ mockFormatFunction.errors = jest.fn();+ mockFormatFunction.splat = jest.fn();+ return {+ format: mockFormatFunction,+ createLogger: jest.fn().mockReturnValue({+ info: jest.fn(),+ warn: jest.fn(),+ debug: jest.fn(),+ error: jest.fn(),+ }),+ transports: {+ Console: jest.fn(),+ DailyRotateFile: jest.fn(),+ },+ addColors: jest.fn(),+ };+});++jest.mock('winston-daily-rotate-file', () => {+ return jest.fn().mockImplementation(() => {+ return {+ level: 'error',+ filename: '../logs/error-%DATE%.log',+ datePattern: 'YYYY-MM-DD',+ zippedArchive: true,+ maxSize: '20m',+ maxFiles: '14d',+ format: 'format',+ };+ });+});+jest.mock('~/config', () => {return {logger: {@@ -8,3 +48,11 @@ jest.mock('~/config', () => {},};});++jest.mock('~/config/parsers', () => {+ return {+ redactMessage: jest.fn(),+ redactFormat: jest.fn(),+ debugTraverse: jest.fn(),+ };+});diff --git a/api/typedefs.js b/api/typedefs.jsindex e40a097634..7bb956c9ae 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -26,6 +26,18 @@* @memberof typedefs*/+/**+ * @exports TMessage+ * @typedef {import('librechat-data-provider').TMessage} TMessage+ * @memberof typedefs+ */++/**+ * @exports FileSources+ * @typedef {import('librechat-data-provider').FileSources} FileSources+ * @memberof typedefs+ */+/*** @exports ImageMetadata* @typedef {Object} ImageMetadatadiff --git a/client/src/common/types.ts b/client/src/common/types.tsindex 17625731ff..1ca169a0c1 100644--- a/client/src/common/types.ts+++ b/client/src/common/types.ts@@ -1,3 +1,4 @@+import { FileSources } from 'librechat-data-provider';import type {TConversation,TMessage,@@ -230,6 +231,7 @@ export interface ExtendedFile {size: number;preview: string;progress: number;+ source?: FileSources;}export type ContextType = { navVisible: boolean; setNavVisible: (visible: boolean) => void };diff --git a/client/src/components/Chat/ChatView.tsx b/client/src/components/Chat/ChatView.tsxindex d582f4b3e9..30a7edc187 100644--- a/client/src/components/Chat/ChatView.tsx+++ b/client/src/components/Chat/ChatView.tsx@@ -6,11 +6,12 @@ import { useChatHelpers, useSSE } from '~/hooks';// import GenerationButtons from './Input/GenerationButtons';import MessagesView from './Messages/MessagesView';// import OptionsBar from './Input/OptionsBar';+import { useGetFiles } from '~/data-provider';+import { buildTree, mapFiles } from '~/utils';import { Spinner } from '~/components/svg';import { ChatContext } from '~/Providers';import Presentation from './Presentation';import ChatForm from './Input/ChatForm';-import { buildTree } from '~/utils';import Landing from './Landing';import Header from './Header';import Footer from './Footer';@@ -21,11 +22,16 @@ function ChatView({ index = 0 }: { index?: number }) {const submissionAtIndex = useRecoilValue(store.submissionByIndex(0));useSSE(submissionAtIndex);+ const { data: fileMap } = useGetFiles({+ select: mapFiles,+ });+const { data: messagesTree = null, isLoading } = useGetMessagesByConvoId(conversationId ?? '', {select: (data) => {- const dataTree = buildTree(data, false);+ const dataTree = buildTree({ messages: data, fileMap });return dataTree?.length === 0 ? null : dataTree ?? null;},+ enabled: !!fileMap,});const chatHelpers = useChatHelpers(index, conversationId);diff --git a/client/src/components/Chat/Input/Files/Images.tsx b/client/src/components/Chat/Input/Files/Images.tsxindex 421f1f69cd..903bd96c71 100644--- a/client/src/components/Chat/Input/Files/Images.tsx+++ b/client/src/components/Chat/Input/Files/Images.tsx@@ -1,5 +1,6 @@import debounce from 'lodash/debounce';import { useState, useEffect, useCallback } from 'react';+import { FileSources } from 'librechat-data-provider';import type { BatchFile } from 'librechat-data-provider';import { useDeleteFilesMutation } from '~/data-provider';import { useSetFilesToDelete } from '~/hooks';@@ -70,13 +71,20 @@ export default function Images({}const deleteFile = (_file: ExtendedFile) => {- const { file_id, progress, temp_file_id = '', filepath = '' } = _file;+ const {+ file_id,+ progress,+ temp_file_id = '',+ filepath = '',+ source = FileSources.local,+ } = _file;if (progress < 1) {return;}- const file = {+ const file: BatchFile = {file_id,filepath,+ source,};setFiles((currentFiles) => {diff --git a/client/src/components/Chat/Messages/Content/MessageContent.tsx b/client/src/components/Chat/Messages/Content/MessageContent.tsxindex 875d4ebb5a..2415259aea 100644--- a/client/src/components/Chat/Messages/Content/MessageContent.tsx+++ b/client/src/components/Chat/Messages/Content/MessageContent.tsx@@ -30,7 +30,7 @@ const ErrorMessage = ({ text }: TText) => {// Display Message Componentconst DisplayMessage = ({ text, isCreatedByUser, message, showCursor }: TDisplayProps) => {const imageFiles = message?.files- ? message.files.filter((file) => file.type.startsWith('image/'))+ ? message.files.filter((file) => file.type && file.type.startsWith('image/')): null;return (<Container>diff --git a/client/src/components/Chat/Presentation.tsx b/client/src/components/Chat/Presentation.tsxindex ea9d77f3b1..af264f1dc4 100644--- a/client/src/components/Chat/Presentation.tsx+++ b/client/src/components/Chat/Presentation.tsx@@ -1,4 +1,5 @@import { useEffect } from 'react';+import { FileSources } from 'librechat-data-provider';import type { ExtendedFile } from '~/common';import { useDragHelpers, useSetFilesToDelete } from '~/hooks';import DragDropOverlay from './Input/Files/DragDropOverlay';@@ -25,6 +26,7 @@ export default function Presentation({ children }: { children: React.ReactNode }.map((file) => ({file_id: file.file_id,filepath: file.filepath as string,+ source: file.source as FileSources,}));if (files.length === 0) {diff --git a/client/src/components/Endpoints/PopoverButtons.tsx b/client/src/components/Endpoints/PopoverButtons.tsxindex 217cce1a52..f763770fca 100644--- a/client/src/components/Endpoints/PopoverButtons.tsx+++ b/client/src/components/Endpoints/PopoverButtons.tsx@@ -18,7 +18,7 @@ export default function PopoverButtons({buttonClass,iconClass = '',}: {- endpoint: EModelEndpoint;+ endpoint: EModelEndpoint | string;buttonClass?: string;iconClass?: string;}) {@@ -32,7 +32,9 @@ export default function PopoverButtons({const buttons: { [key: string]: TPopoverButton[] } = {google: [{- label: (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) + localize('com_endpoint_examples'),+ label:+ (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) ++ localize('com_endpoint_examples'),buttonClass: isCodeChat ? 'disabled' : '',handler: triggerExamples,icon: <MessagesSquared className={cn('mr-1 w-[14px]', iconClass)} />,@@ -40,7 +42,10 @@ export default function PopoverButtons({],gptPlugins: [{- label: localize('com_endpoint_show_what_settings', showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent')),+ label: localize(+ 'com_endpoint_show_what_settings',+ showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent'),+ ),buttonClass: '',handler: () => setShowAgentSettings((prev) => !prev),icon: <GPTIcon className={cn('mr-1 w-[14px]', iconClass)} size={24} />,diff --git a/client/src/components/Endpoints/Settings/OpenAI.tsx b/client/src/components/Endpoints/Settings/OpenAI.tsxindex 77b224c8d3..652d46c586 100644--- a/client/src/components/Endpoints/Settings/OpenAI.tsx+++ b/client/src/components/Endpoints/Settings/OpenAI.tsx@@ -1,11 +1,13 @@import TextareaAutosize from 'react-textarea-autosize';+import { ImageDetail, imageDetailNumeric, imageDetailValue } from 'librechat-data-provider';import {- SelectDropDown,Input,Label,+ Switch,Slider,- InputNumber,HoverCard,+ InputNumber,+ SelectDropDown,HoverCardTrigger,} from '~/components/ui';import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/';@@ -20,6 +22,8 @@ export default function Settings({ conversation, setOption, models, readonly }:return null;}const {+ endpoint,+ endpointType,model,chatGptLabel,promptPrefix,@@ -27,6 +31,8 @@ export default function Settings({ conversation, setOption, models, readonly }:top_p: topP,frequency_penalty: freqP,presence_penalty: presP,+ resendImages,+ imageDetail,} = conversation;const setModel = setOption('model');const setChatGptLabel = setOption('chatGptLabel');@@ -35,6 +41,10 @@ export default function Settings({ conversation, setOption, models, readonly }:const setTopP = setOption('top_p');const setFreqP = setOption('frequency_penalty');const setPresP = setOption('presence_penalty');+ const setResendImages = setOption('resendImages');+ const setImageDetail = setOption('imageDetail');++ const optionEndpoint = endpointType ?? endpoint;return (<div className="grid grid-cols-5 gap-6">@@ -126,7 +136,7 @@ export default function Settings({ conversation, setOption, models, readonly }:className="flex h-4 w-full"/></HoverCardTrigger>- <OptionHover endpoint={conversation?.endpoint ?? ''} type="temp" side={ESide.Left} />+ <OptionHover endpoint={optionEndpoint ?? ''} type="temp" side={ESide.Left} /></HoverCard><HoverCard openDelay={300}><HoverCardTrigger className="grid w-full items-center gap-2">@@ -164,7 +174,7 @@ export default function Settings({ conversation, setOption, models, readonly }:className="flex h-4 w-full"/></HoverCardTrigger>- <OptionHover endpoint={conversation?.endpoint ?? ''} type="topp" side={ESide.Left} />+ <OptionHover endpoint={optionEndpoint ?? ''} type="topp" side={ESide.Left} /></HoverCard><HoverCard openDelay={300}>@@ -203,7 +213,7 @@ export default function Settings({ conversation, setOption, models, readonly }:className="flex h-4 w-full"/></HoverCardTrigger>- <OptionHover endpoint={conversation?.endpoint ?? ''} type="freq" side={ESide.Left} />+ <OptionHover endpoint={optionEndpoint ?? ''} type="freq" side={ESide.Left} /></HoverCard><HoverCard openDelay={300}>@@ -242,8 +252,66 @@ export default function Settings({ conversation, setOption, models, readonly }:className="flex h-4 w-full"/></HoverCardTrigger>- <OptionHover endpoint={conversation?.endpoint ?? ''} type="pres" side={ESide.Left} />+ <OptionHover endpoint={optionEndpoint ?? ''} type="pres" side={ESide.Left} /></HoverCard>+ <div className="w-full">+ <div className="mb-2 flex w-full justify-between gap-2">+ <label+ htmlFor="resend-images"+ className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"+ >+ <small>{localize('com_endpoint_plug_resend_images')}</small>+ </label>+ <label+ htmlFor="image-detail-value"+ className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"+ >+ <small>Image Detail</small>+ </label>+ <Input+ id="image-detail-value"+ disabled={true}+ value={imageDetail ?? ImageDetail.auto}+ className={cn(+ defaultTextProps,+ optionText,+ 'flex rounded-md bg-transparent py-2 text-xs focus:outline-none focus:ring-2 focus:ring-slate-400 focus:ring-offset-2 dark:border-slate-700',+ 'pointer-events-none max-h-5 w-12 border-0 group-hover/temp:border-gray-200',+ )}+ />+ </div>+ <div className="flex w-full justify-between gap-2">+ <HoverCard openDelay={500}>+ <HoverCardTrigger>+ <Switch+ id="resend-images"+ checked={resendImages ?? false}+ onCheckedChange={(checked: boolean) => setResendImages(checked)}+ disabled={readonly}+ className="flex"+ />+ <OptionHover endpoint={optionEndpoint ?? ''} type="resend" side={ESide.Bottom} />+ </HoverCardTrigger>+ </HoverCard>+ <HoverCard openDelay={500}>+ <HoverCardTrigger className="flex w-[52%] md:w-[125px]">+ <Slider+ id="image-detail-slider"+ disabled={readonly}+ value={[+ imageDetailNumeric[imageDetail ?? ''] ?? imageDetailNumeric[ImageDetail.auto],+ ]}+ onValueChange={(value) => setImageDetail(imageDetailValue[value[0]])}+ doubleClickHandler={() => setImageDetail(ImageDetail.auto)}+ max={2}+ min={0}+ step={1}+ />+ <OptionHover endpoint={optionEndpoint ?? ''} type="detail" side={ESide.Bottom} />+ </HoverCardTrigger>+ </HoverCard>+ </div>+ </div></div></div>);diff --git a/client/src/components/Endpoints/Settings/OptionHover.tsx b/client/src/components/Endpoints/Settings/OptionHover.tsxindex 2648a06184..f6af64c149 100644--- a/client/src/components/Endpoints/Settings/OptionHover.tsx+++ b/client/src/components/Endpoints/Settings/OptionHover.tsx@@ -15,6 +15,8 @@ const openAI = {topp: 'com_endpoint_openai_topp',freq: 'com_endpoint_openai_freq',pres: 'com_endpoint_openai_pres',+ resend: 'com_endpoint_openai_resend',+ detail: 'com_endpoint_openai_detail',};const types = {@@ -47,7 +49,7 @@ function OptionHover({ endpoint, type, side }: TOptionHoverProps) {}return (<HoverCardPortal>- <HoverCardContent side={side} className="w-80 ">+ <HoverCardContent side={side} className="z-[80] w-80"><div className="space-y-2"><p className="text-sm text-gray-600 dark:text-gray-300">{localize(text)}</p></div>diff --git a/client/src/components/Nav/ExportConversation/ExportModal.jsx b/client/src/components/Nav/ExportConversation/ExportModal.jsxindex e4f292a78a..d2cfb49c1e 100644--- a/client/src/components/Nav/ExportConversation/ExportModal.jsx+++ b/client/src/components/Nav/ExportConversation/ExportModal.jsx@@ -24,7 +24,7 @@ export default function ExportModal({ open, onOpenChange, conversation }) {const { data: messagesTree = null } = useGetMessagesByConvoId(conversation.conversationId ?? '', {select: (data) => {- const dataTree = buildTree(data, false);+ const dataTree = buildTree({ messages: data });return dataTree?.length === 0 ? null : dataTree ?? null;},});diff --git a/client/src/data-provider/mutations.ts b/client/src/data-provider/mutations.tsindex 236c66a947..e607818cc7 100644--- a/client/src/data-provider/mutations.ts+++ b/client/src/data-provider/mutations.ts@@ -1,7 +1,7 @@import { useMutation, useQueryClient } from '@tanstack/react-query';import type { UseMutationResult } from '@tanstack/react-query';import type {- FileUploadResponse,+ TFileUpload,UploadMutationOptions,FileUploadBody,DeleteFilesResponse,@@ -23,7 +23,7 @@ import store from '~/store';export const useUploadImageMutation = (options?: UploadMutationOptions,): UseMutationResult<- FileUploadResponse, // response data+ TFileUpload, // response dataunknown, // errorFileUploadBody, // requestunknown // contextdiff --git a/client/src/data-provider/queries.ts b/client/src/data-provider/queries.tsindex 4144b64812..2ae6aaf81b 100644--- a/client/src/data-provider/queries.ts+++ b/client/src/data-provider/queries.ts@@ -1,6 +1,18 @@-import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';import { QueryKeys, dataService } from 'librechat-data-provider';-import type { TPreset } from 'librechat-data-provider';+import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';+import type { TPreset, TFile } from 'librechat-data-provider';++export const useGetFiles = <TData = TFile[] | boolean>(+ config?: UseQueryOptions<TFile[], unknown, TData>,+): QueryObserverResult<TData, unknown> => {+ return useQuery<TFile[], unknown, TData>([QueryKeys.files], () => dataService.getFiles(), {+ refetchOnWindowFocus: false,+ refetchOnReconnect: false,+ refetchOnMount: false,+ ...config,+ });+};+export const useGetPresetsQuery = (config?: UseQueryOptions<TPreset[]>,): QueryObserverResult<TPreset[], unknown> => {diff --git a/client/src/hooks/useFileHandling.ts b/client/src/hooks/useFileHandling.tsindex 21126a98f8..bd6874336e 100644--- a/client/src/hooks/useFileHandling.ts+++ b/client/src/hooks/useFileHandling.ts@@ -1,6 +1,9 @@import { v4 } from 'uuid';import debounce from 'lodash/debounce';+import { QueryKeys } from 'librechat-data-provider';+import { useQueryClient } from '@tanstack/react-query';import { useState, useEffect, useCallback } from 'react';+import type { TFile } from 'librechat-data-provider';import type { ExtendedFile } from '~/common';import { useToastContext } from '~/Providers/ToastContext';import { useChatContext } from '~/Providers/ChatContext';@@ -16,6 +19,7 @@ const totalSizeLimit = maxSize * 1024 * 1024; // 25 MBconst supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];const useFileHandling = () => {+ const queryClient = useQueryClient();const { showToast } = useToastContext();const [errors, setErrors] = useState<string[]>([]);const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);@@ -116,6 +120,9 @@ const useFileHandling = () => {filepath: data.filepath,});+ const _files = queryClient.getQueryData<TFile[]>([QueryKeys.files]) ?? [];+ queryClient.setQueryData([QueryKeys.files], [..._files, data]);+setTimeout(() => {updateFileById(data.temp_file_id, {progress: 1,@@ -126,6 +133,7 @@ const useFileHandling = () => {height: data.height,width: data.width,filename: data.filename,+ source: data.source,});}, 300);},diff --git a/client/src/hooks/useNewConvo.ts b/client/src/hooks/useNewConvo.tsindex 897636e587..d6c3bba8c3 100644--- a/client/src/hooks/useNewConvo.ts+++ b/client/src/hooks/useNewConvo.ts@@ -1,4 +1,5 @@import { useCallback } from 'react';+import { FileSources } from 'librechat-data-provider';import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';import {useSetRecoilState,@@ -122,10 +123,11 @@ const useNewConvo = (index = 0) => {if (conversation.conversationId === 'new' && !modelsData) {const filesToDelete = Array.from(files.values())- .filter((file) => file.filepath)+ .filter((file) => file.filepath && file.source).map((file) => ({file_id: file.file_id,filepath: file.filepath as string,+ source: file.source as FileSources, // Ensure that the source is of type FileSources}));setFiles(new Map());diff --git a/client/src/localization/languages/Eng.tsx b/client/src/localization/languages/Eng.tsxindex f084cf50f8..52749713e3 100644--- a/client/src/localization/languages/Eng.tsx+++ b/client/src/localization/languages/Eng.tsx@@ -162,6 +162,10 @@ export default {'Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model\'s likelihood to repeat the same line verbatim.',com_endpoint_openai_pres:'Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model\'s likelihood to talk about new topics.',+ com_endpoint_openai_resend:+ 'Resend all previously attached images. Note: this can significantly increase token cost and you may experience errors with many image attachments.',+ com_endpoint_openai_detail:+ 'The resolution for Vision requests. "Low" is cheaper and faster, "High" is more detailed and expensive, and "Auto" will automatically choose between the two based on the image resolution.',com_endpoint_openai_custom_name_placeholder: 'Set a custom name for ChatGPT',com_endpoint_openai_prompt_prefix_placeholder:'Set custom instructions to include in System Message. Default: none',@@ -177,6 +181,7 @@ export default {com_endpoint_frequency_penalty: 'Frequency Penalty',com_endpoint_presence_penalty: 'Presence Penalty',com_endpoint_plug_use_functions: 'Use Functions',+ com_endpoint_plug_resend_images: 'Resend Images',com_endpoint_plug_skip_completion: 'Skip Completion',com_endpoint_disabled_with_tools: 'disabled with tools',com_endpoint_disabled_with_tools_placeholder: 'Disabled with Tools Selected',diff --git a/client/src/routes/ChatRoute.tsx b/client/src/routes/ChatRoute.tsxindex 8cca53233b..aaf0e5ab8c 100644--- a/client/src/routes/ChatRoute.tsx+++ b/client/src/routes/ChatRoute.tsx@@ -6,7 +6,7 @@ import {useGetModelsQuery,useGetEndpointsQuery,} from 'librechat-data-provider/react-query';-import { TPreset } from 'librechat-data-provider';+import type { TPreset } from 'librechat-data-provider';import { useNewConvo, useConfigOverride } from '~/hooks';import ChatView from '~/components/Chat/ChatView';import useAuthRedirect from './useAuthRedirect';@@ -15,6 +15,7 @@ import store from '~/store';export default function ChatRoute() {const index = 0;+useConfigOverride();const { conversationId } = useParams();const { conversation } = store.useCreateConversationAtom(index);diff --git a/client/src/routes/Root.tsx b/client/src/routes/Root.tsxindex 7da118ba97..28224de3fc 100644--- a/client/src/routes/Root.tsx+++ b/client/src/routes/Root.tsx@@ -1,11 +1,12 @@/* eslint-disable react-hooks/exhaustive-deps */import { useEffect, useState } from 'react';-import { useRecoilValue, useSetRecoilState } from 'recoil';import { Outlet, useLocation } from 'react-router-dom';+import { useRecoilValue, useSetRecoilState } from 'recoil';import { useGetModelsQuery, useGetSearchEnabledQuery } from 'librechat-data-provider/react-query';import type { ContextType } from '~/common';import { useAuthContext, useServerStream, useConversation } from '~/hooks';import { Nav, MobileNav } from '~/components/Nav';+import { useGetFiles } from '~/data-provider';import store from '~/store';export default function Root() {@@ -24,6 +25,7 @@ export default function Root() {const setIsSearchEnabled = useSetRecoilState(store.isSearchEnabled);const setModelsConfig = useSetRecoilState(store.modelsConfig);+ useGetFiles({ enabled: isAuthenticated });const searchEnabledQuery = useGetSearchEnabledQuery({ enabled: isAuthenticated });const modelsQuery = useGetModelsQuery({ enabled: isAuthenticated && modelsQueryEnabled });diff --git a/client/src/store/conversation.ts b/client/src/store/conversation.tsindex a575ac385d..ee12d71496 100644--- a/client/src/store/conversation.ts+++ b/client/src/store/conversation.ts@@ -18,7 +18,7 @@ const messages = atom<TMessagesAtom>({const messagesTree = selector({key: 'messagesTree',get: ({ get }) => {- return buildTree(get(messages), false);+ return buildTree({ messages: get(messages) });},});diff --git a/client/src/store/search.ts b/client/src/store/search.tsindex 4f4099b473..b687058e87 100644--- a/client/src/store/search.ts+++ b/client/src/store/search.ts@@ -20,7 +20,7 @@ const searchResultMessages = atom<TMessage[] | null>({const searchResultMessagesTree = selector({key: 'searchResultMessagesTree',get: ({ get }) => {- return buildTree(get(searchResultMessages), true);+ return buildTree({ messages: get(searchResultMessages), groupAll: true });},});diff --git a/client/src/utils/buildTree.ts b/client/src/utils/buildTree.tsindex 23b74a11a0..637801b548 100644--- a/client/src/utils/buildTree.ts+++ b/client/src/utils/buildTree.ts@@ -1,11 +1,19 @@-import { TMessage } from 'librechat-data-provider';+import { TFile, TMessage } from 'librechat-data-provider';const even ='w-full border-b border-black/10 dark:border-gray-900/50 text-gray-800 bg-white dark:text-gray-100 group dark:bg-gray-800 hover:bg-gray-100/25 hover:text-gray-700 dark:hover:bg-gray-900 dark:hover:text-gray-200';const odd ='w-full border-b border-black/10 bg-gray-50 dark:border-gray-900/50 text-gray-800 dark:text-gray-100 group bg-gray-100 dark:bg-gray-1000 hover:bg-gray-100/40 hover:text-gray-700 dark:hover:bg-[#3b3d49] dark:hover:text-gray-200';-export default function buildTree(messages: TMessage[] | null, groupAll = false) {+export default function buildTree({+ messages,+ fileMap,+ groupAll = false,+}: {+ messages: TMessage[] | null;+ fileMap?: Record<string, TFile>;+ groupAll?: boolean;+}) {if (messages === null) {return null;}@@ -21,6 +29,12 @@ export default function buildTree(messages: TMessage[] | null, groupAll = false)messages.forEach((message) => {messageMap[message.messageId] = { ...message, children: [] };+ if (message.files && fileMap) {+ messageMap[message.messageId].files = message.files.map(+ (file) => fileMap[file.file_id] ?? file,+ );+ }+const parentMessage = messageMap[message.parentMessageId ?? ''];if (parentMessage) {parentMessage.children.push(messageMap[message.messageId]);diff --git a/client/src/utils/files.ts b/client/src/utils/files.tsnew file mode 100644index 0000000000..4615029fff--- /dev/null+++ b/client/src/utils/files.ts@@ -0,0 +1,12 @@+import type { TFile } from 'librechat-data-provider';++/** Maps Files by `file_id` for quick lookup */+export function mapFiles(files: TFile[]) {+ const fileMap = {} as Record<string, TFile>;++ for (const file of files) {+ fileMap[file.file_id] = file;+ }++ return fileMap;+}diff --git a/client/src/utils/index.ts b/client/src/utils/index.tsindex bf7ab065c4..12a32faf63 100644--- a/client/src/utils/index.ts+++ b/client/src/utils/index.ts@@ -1,4 +1,5 @@export * from './json';+export * from './files';export * from './presets';export * from './languages';export * from './endpoints';diff --git a/client/src/utils/presets.ts b/client/src/utils/presets.tsindex ee86087d87..45a7cd21fd 100644--- a/client/src/utils/presets.ts+++ b/client/src/utils/presets.ts@@ -12,6 +12,8 @@ export const getPresetIcon = (preset: TPreset, Icon) => {});};+type TEndpoints = Array<string | EModelEndpoint>;+export const getPresetTitle = (preset: TPreset) => {const {endpoint,@@ -26,12 +28,16 @@ export const getPresetTitle = (preset: TPreset) => {let modelInfo = model || '';let label = '';- if (- endpoint &&- [EModelEndpoint.azureOpenAI, EModelEndpoint.openAI, EModelEndpoint.custom].includes(endpoint)- ) {+ const usesChatGPTLabel: TEndpoints = [+ EModelEndpoint.azureOpenAI,+ EModelEndpoint.openAI,+ EModelEndpoint.custom,+ ];+ const usesModelLabel: TEndpoints = [EModelEndpoint.google, EModelEndpoint.anthropic];++ if (endpoint && usesChatGPTLabel.includes(endpoint)) {label = chatGptLabel || '';- } else if (endpoint && [EModelEndpoint.google, EModelEndpoint.anthropic].includes(endpoint)) {+ } else if (endpoint && usesModelLabel.includes(endpoint)) {label = modelLabel || '';} else if (endpoint === EModelEndpoint.bingAI) {modelInfo = jailbreak ? 'Sydney' : modelInfo;diff --git a/docs/features/firebase.md b/docs/features/firebase.mdindex fc94b3bce9..e48ab9a2e7 100644--- a/docs/features/firebase.md+++ b/docs/features/firebase.md@@ -100,4 +100,20 @@ FIREBASE_APP_ID=1:your_app_id #appId- Publish your updated rules-\ No newline at end of file+++### Configure `fileStrategy` in `librechat.yaml`++Finally, to enable the app use Firebase, you must set the following in your `librechat.yaml` config file.++```yaml+ version: 1.0.1+ cache: true+ fileStrategy: "firebase" # This is the field and value you need to add+ endpoints:+ custom:+ - name: "Mistral"+ # Rest of file omitted+```++For more information about the `librechat.yaml` config file, [see the guide here](../install/configuration/custom_config.md).\ No newline at end of filediff --git a/docs/features/index.md b/docs/features/index.mdindex 9bfda2be25..e025a83d2b 100644--- a/docs/features/index.md+++ b/docs/features/index.md@@ -8,6 +8,7 @@ weight: 2---+* 🤖[Custom Endpoints](../install/configuration/custom_config.md)* 🔌[Plugins](./plugins/index.md)* 🔌 [Introduction](./plugins/introduction.md)* 🛠️ [Make Your Own](./plugins/make_your_own.md)diff --git a/docs/install/configuration/custom_config.md b/docs/install/configuration/custom_config.mdindex 0e62330977..b9802fe051 100644--- a/docs/install/configuration/custom_config.md+++ b/docs/install/configuration/custom_config.md@@ -69,20 +69,26 @@ docker-compose up # no need to rebuild**Note:** Fields not specifically mentioned as required are optional.-### 1. Version+### Version- **Key**: `version`- **Type**: String- **Description**: Specifies the version of the configuration file.-- **Example**: `version: 1.0.0`+- **Example**: `version: 1.0.1`- **Required**-### 2. Cache Settings+### Cache Settings- **Key**: `cache`- **Type**: Boolean- **Description**: Toggles caching on or off. Set to `true` to enable caching.- **Example**: `cache: true`-### 3. Endpoints+### File Strategy+- **Key**: `fileStrategy`+- **Type**: String = "local" | ["firebase"](../../features/firebase.md)+- **Description**: Determines where to save user uploaded/generated files. Defaults to `"local"` if omitted.+- **Example**: `fileStrategy: "firebase"`++### Endpoints- **Key**: `endpoints`- **Type**: Object- **Description**: Defines custom API endpoints for the application.@@ -290,7 +296,7 @@ Custom endpoints share logic with the OpenAI endpoint, and thus have default par## Example Config```yaml-version: 1.0.0+version: 1.0.1cache: trueendpoints:custom:diff --git a/librechat.example.yaml b/librechat.example.yamlindex b2a9bbaf93..047cfbcb30 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -1,5 +1,5 @@# Configuration version (required)-version: 1.0.0+version: 1.0.1# Cache settings: Set to true to enable cachingcache: truediff --git a/packages/data-provider/package.json b/packages/data-provider/package.jsonindex c0b0a05209..9148305ce1 100644--- a/packages/data-provider/package.json+++ b/packages/data-provider/package.json@@ -1,6 +1,6 @@{"name": "librechat-data-provider",- "version": "0.3.5",+ "version": "0.3.6","description": "data services for librechat apps","main": "dist/index.js","module": "dist/index.es.js",diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 7b31508cc5..a7cc732d16 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1,5 +1,8 @@import { z } from 'zod';import { EModelEndpoint, eModelEndpointSchema } from './schemas';+import { FileSources } from './types/files';++export const fileSourceSchema = z.nativeEnum(FileSources);export const endpointSchema = z.object({name: z.string().refine((value) => !eModelEndpointSchema.safeParse(value).success, {@@ -25,6 +28,7 @@ export const endpointSchema = z.object({export const configSchema = z.object({version: z.string(),cache: z.boolean(),+ fileStrategy: fileSourceSchema.optional(),endpoints: z.object({custom: z.array(endpointSchema.partial()),@@ -184,3 +188,27 @@ export enum AuthKeys {*/GOOGLE_API_KEY = 'GOOGLE_API_KEY',}++/**+ * Enum for Image Detail Cost.+ *+ * **Low Res Fixed Cost:** `85`+ *+ * **High Res Calculation:**+ *+ * Number of `512px` Tiles * `170` + `85` (Additional Cost)+ */+export enum ImageDetailCost {+ /**+ * Low resolution is a fixed value.+ */+ LOW = 85,+ /**+ * High resolution Cost Per Tile+ */+ HIGH = 170,+ /**+ * Additional Cost added to High Resolution Total Cost+ */+ ADDITIONAL = 85,+}diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.tsindex 74ad766575..f6b4221404 100644--- a/packages/data-provider/src/data-service.ts+++ b/packages/data-provider/src/data-service.ts@@ -193,7 +193,11 @@ export const listAssistants = (/* Files */-export const uploadImage = (data: FormData): Promise<f.FileUploadResponse> => {+export const getFiles = (): Promise<f.TFile[]> => {+ return request.get(endpoints.files());+};++export const uploadImage = (data: FormData): Promise<f.TFileUpload> => {return request.postMultiPart(endpoints.images(), data);};diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.tsindex ec150d9587..b0368f5b96 100644--- a/packages/data-provider/src/keys.ts+++ b/packages/data-provider/src/keys.ts@@ -16,6 +16,7 @@ export enum QueryKeys {assistants = 'assistants',assistant = 'assistant',endpointsConfigOverride = 'endpointsConfigOverride',+ files = 'files',}export enum MutationKeys {diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.tsindex caa951fa6a..25cf3825c7 100644--- a/packages/data-provider/src/schemas.ts+++ b/packages/data-provider/src/schemas.ts@@ -52,6 +52,26 @@ export const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);export const extendedModelEndpointSchema = z.union([eModelEndpointSchema, z.string()]);+export enum ImageDetail {+ low = 'low',+ auto = 'auto',+ high = 'high',+}++export const imageDetailNumeric = {+ [ImageDetail.low]: 0,+ [ImageDetail.auto]: 1,+ [ImageDetail.high]: 2,+};++export const imageDetailValue = {+ 0: ImageDetail.low,+ 1: ImageDetail.auto,+ 2: ImageDetail.high,+};++export const eImageDetailSchema = z.nativeEnum(ImageDetail);+export const tPluginAuthConfigSchema = z.object({authField: z.string(),label: z.string(),@@ -140,8 +160,8 @@ export type TMessage = z.input<typeof tMessageSchema> & {plugin?: TResPlugin | null;plugins?: TResPlugin[];files?: {- type: string;file_id: string;+ type?: string;filename?: string;preview?: string;filepath?: string;@@ -184,6 +204,9 @@ export const tConversationSchema = z.object({toneStyle: z.string().nullable().optional(),maxOutputTokens: z.number().optional(),agentOptions: tAgentOptionsSchema.nullable().optional(),+ /* vision */+ resendImages: z.boolean().optional(),+ imageDetail: eImageDetailSchema.optional(),/* assistant */assistant_id: z.string().optional(),thread_id: z.string().optional(),@@ -234,6 +257,8 @@ export const openAISchema = tConversationSchematop_p: true,presence_penalty: true,frequency_penalty: true,+ resendImages: true,+ imageDetail: true,}).transform((obj) => ({...obj,@@ -244,6 +269,8 @@ export const openAISchema = tConversationSchematop_p: obj.top_p ?? 1,presence_penalty: obj.presence_penalty ?? 0,frequency_penalty: obj.frequency_penalty ?? 0,+ resendImages: obj.resendImages ?? false,+ imageDetail: obj.imageDetail ?? ImageDetail.auto,})).catch(() => ({model: 'gpt-3.5-turbo',@@ -253,6 +280,8 @@ export const openAISchema = tConversationSchematop_p: 1,presence_penalty: 0,frequency_penalty: 0,+ resendImages: false,+ imageDetail: ImageDetail.auto,}));export const googleSchema = tConversationSchema@@ -455,6 +484,8 @@ export const compactOpenAISchema = tConversationSchematop_p: true,presence_penalty: true,frequency_penalty: true,+ resendImages: true,+ imageDetail: true,}).transform((obj: Partial<TConversation>) => {const newObj: Partial<TConversation> = { ...obj };@@ -473,6 +504,12 @@ export const compactOpenAISchema = tConversationSchemaif (newObj.frequency_penalty === 0) {delete newObj.frequency_penalty;}+ if (newObj.resendImages !== true) {+ delete newObj.resendImages;+ }+ if (newObj.imageDetail === ImageDetail.auto) {+ delete newObj.imageDetail;+ }return removeNullishValues(newObj);})diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.tsindex 95f329f652..dadf5cc640 100644--- a/packages/data-provider/src/types/files.ts+++ b/packages/data-provider/src/types/files.ts@@ -1,13 +1,25 @@-export type FileUploadResponse = {+export enum FileSources {+ local = 'local',+ firebase = 'firebase',+ openai = 'openai',+ s3 = 's3',+}++export type TFile = {message: string;file_id: string;- temp_file_id: string;filepath: string;filename: string;type: string;size: number;- height: number;- width: number;+ temp_file_id?: string;+ source?: FileSources;+ height?: number;+ width?: number;+};++export type TFileUpload = TFile & {+ temp_file_id: string;};export type AvatarUploadResponse = {@@ -20,7 +32,7 @@ export type FileUploadBody = {};export type UploadMutationOptions = {- onSuccess?: (data: FileUploadResponse, variables: FileUploadBody, context?: unknown) => void;+ onSuccess?: (data: TFileUpload, variables: FileUploadBody, context?: unknown) => void;onMutate?: (variables: FileUploadBody) => void | Promise<unknown>;onError?: (error: unknown, variables: FileUploadBody, context?: unknown) => void;};@@ -39,6 +51,7 @@ export type DeleteFilesResponse = {export type BatchFile = {file_id: string;filepath: string;+ source: FileSources;};export type DeleteFilesBody = {