Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🚀 Feat: Streamline File Strategies & GPT-4-Vision Settings (#1535)

Danny Avilad20970f5c5c9
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js
index 5b7ea103c2..aa39084b9f 100644
--- a/api/app/clients/BaseClient.js
+++ b/api/app/clients/BaseClient.js
@@ -46,6 +46,10 @@ class BaseClient {
logger.debug('`[BaseClient] recordTokenUsage` not implemented.', response);
}
+ async addPreviousAttachments(messages) {
+ return messages;
+ }
+
async recordTokenUsage({ promptTokens, completionTokens }) {
logger.debug('`[BaseClient] recordTokenUsage` not implemented.', {
promptTokens,
@@ -484,20 +488,22 @@ class BaseClient {
mapMethod = this.getMessageMapMethod();
}
- const orderedMessages = this.constructor.getMessagesForConversation({
+ let _messages = this.constructor.getMessagesForConversation({
messages,
parentMessageId,
mapMethod,
});
+ _messages = await this.addPreviousAttachments(_messages);
+
if (!this.shouldSummarize) {
- return orderedMessages;
+ return _messages;
}
// Find the latest message with a 'summary' property
- for (let i = orderedMessages.length - 1; i >= 0; i--) {
- if (orderedMessages[i]?.summary) {
- this.previous_summary = orderedMessages[i];
+ for (let i = _messages.length - 1; i >= 0; i--) {
+ if (_messages[i]?.summary) {
+ this.previous_summary = _messages[i];
break;
}
}
@@ -512,7 +518,7 @@ class BaseClient {
});
}
- return orderedMessages;
+ return _messages;
}
async saveMessageToDatabase(message, endpointOptions, user = null) {
@@ -618,6 +624,11 @@ class BaseClient {
* An additional 3 tokens need to be added for assistant label priming after all messages have been counted.
* In our implementation, this is accounted for in the getMessagesWithinTokenLimit method.
*
+ * The content parts example was adapted from the following example:
+ * https://github.com/openai/openai-cookbook/pull/881/files
+ *
+ * Note: image token calculation is to be done elsewhere where we have access to the image metadata
+ *
* @param {Object} message
*/
getTokenCountForMessage(message) {
@@ -631,11 +642,18 @@ class BaseClient {
}
const processValue = (value) => {
- if (typeof value === 'object' && value !== null) {
- for (let [nestedKey, nestedValue] of Object.entries(value)) {
- if (nestedKey === 'image_url' || nestedValue === 'image_url') {
+ if (Array.isArray(value)) {
+ for (let item of value) {
+ if (!item || !item.type || item.type === 'image_url') {
continue;
}
+
+ const nestedValue = item[item.type];
+
+ if (!nestedValue) {
+ continue;
+ }
+
processValue(nestedValue);
}
} else {
diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js
index 1c22d6f7d4..f832b3ca13 100644
--- a/api/app/clients/OpenAIClient.js
+++ b/api/app/clients/OpenAIClient.js
@@ -1,6 +1,6 @@
const OpenAI = require('openai');
const { HttpsProxyAgent } = require('https-proxy-agent');
-const { getResponseSender } = require('librechat-data-provider');
+const { getResponseSender, ImageDetailCost, ImageDetail } = require('librechat-data-provider');
const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
const { encodeAndFormat, validateVisionModel } = require('~/server/services/Files/images');
const { getModelMaxTokens, genAzureChatCompletion, extractBaseURL } = require('~/utils');
@@ -8,8 +8,9 @@ const { truncateText, formatMessage, CUT_OFF_PROMPT } = require('./prompts');
const { handleOpenAIErrors } = require('./tools/util');
const spendTokens = require('~/models/spendTokens');
const { createLLM, RunManager } = require('./llm');
-const { isEnabled } = require('~/server/utils');
const ChatGPTClient = require('./ChatGPTClient');
+const { isEnabled } = require('~/server/utils');
+const { getFiles } = require('~/models/File');
const { summaryBuffer } = require('./memory');
const { runTitleChain } = require('./chains');
const { tokenSplit } = require('./document');
@@ -76,16 +77,7 @@ class OpenAIClient extends BaseClient {
};
}
- this.isVisionModel = validateVisionModel(this.modelOptions.model);
-
- if (this.options.attachments && !this.isVisionModel) {
- this.modelOptions.model = 'gpt-4-vision-preview';
- this.isVisionModel = true;
- }
-
- if (this.isVisionModel) {
- delete this.modelOptions.stop;
- }
+ this.checkVisionRequest(this.options.attachments);
const { OPENROUTER_API_KEY, OPENAI_FORCE_PROMPT } = process.env ?? {};
if (OPENROUTER_API_KEY && !this.azure) {
@@ -204,6 +196,27 @@ class OpenAIClient extends BaseClient {
return this;
}
+ /**
+ *
+ * Checks if the model is a vision model based on request attachments and sets the appropriate options:
+ * - Sets `this.modelOptions.model` to `gpt-4-vision-preview` if the request is a vision request.
+ * - Sets `this.isVisionModel` to `true` if vision request.
+ * - Deletes `this.modelOptions.stop` if vision request.
+ * @param {Array<Promise<MongoFile[]> | MongoFile[]> | Record<string, MongoFile[]>} attachments
+ */
+ checkVisionRequest(attachments) {
+ this.isVisionModel = validateVisionModel(this.modelOptions.model);
+
+ if (attachments && !this.isVisionModel) {
+ this.modelOptions.model = 'gpt-4-vision-preview';
+ this.isVisionModel = true;
+ }
+
+ if (this.isVisionModel) {
+ delete this.modelOptions.stop;
+ }
+ }
+
setupTokens() {
if (this.isChatCompletion) {
this.startToken = '||>';
@@ -288,7 +301,11 @@ class OpenAIClient extends BaseClient {
tokenizerCallsCount++;
}
- // Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+ /**
+ * Returns the token count of a given text. It also checks and resets the tokenizers if necessary.
+ * @param {string} text - The text to get the token count for.
+ * @returns {number} The token count of the given text.
+ */
getTokenCount(text) {
this.resetTokenizersIfNecessary();
try {
@@ -301,10 +318,33 @@ class OpenAIClient extends BaseClient {
}
}
+ /**
+ * Calculate the token cost for an image based on its dimensions and detail level.
+ *
+ * @param {Object} image - The image object.
+ * @param {number} image.width - The width of the image.
+ * @param {number} image.height - The height of the image.
+ * @param {'low'|'high'|string|undefined} [image.detail] - The detail level ('low', 'high', or other).
+ * @returns {number} The calculated token cost.
+ */
+ calculateImageTokenCost({ width, height, detail }) {
+ if (detail === 'low') {
+ return ImageDetailCost.LOW;
+ }
+
+ // Calculate the number of 512px squares
+ const numSquares = Math.ceil(width / 512) * Math.ceil(height / 512);
+
+ // Default to high detail cost calculation
+ return numSquares * ImageDetailCost.HIGH + ImageDetailCost.ADDITIONAL;
+ }
+
getSaveOptions() {
return {
chatGptLabel: this.options.chatGptLabel,
promptPrefix: this.options.promptPrefix,
+ resendImages: this.options.resendImages,
+ imageDetail: this.options.imageDetail,
...this.modelOptions,
};
}
@@ -317,6 +357,69 @@ class OpenAIClient extends BaseClient {
};
}
+ /**
+ *
+ * @param {TMessage[]} _messages
+ * @returns {TMessage[]}
+ */
+ async addPreviousAttachments(_messages) {
+ if (!this.options.resendImages) {
+ return _messages;
+ }
+
+ /**
+ *
+ * @param {TMessage} message
+ */
+ const processMessage = async (message) => {
+ if (!this.message_file_map) {
+ /** @type {Record<string, MongoFile[]> */
+ this.message_file_map = {};
+ }
+
+ const fileIds = message.files.map((file) => file.file_id);
+ const files = await getFiles({
+ file_id: { $in: fileIds },
+ });
+
+ await this.addImageURLs(message, files);
+
+ this.message_file_map[message.messageId] = files;
+ return message;
+ };
+
+ const promises = [];
+
+ for (const message of _messages) {
+ if (!message.files) {
+ promises.push(message);
+ continue;
+ }
+
+ promises.push(processMessage(message));
+ }
+
+ const messages = await Promise.all(promises);
+
+ this.checkVisionRequest(this.message_file_map);
+ return messages;
+ }
+
+ /**
+ *
+ * Adds image URLs to the message object and returns the files
+ *
+ * @param {TMessage[]} messages
+ * @param {MongoFile[]} files
+ * @returns {Promise<MongoFile[]>}
+ */
+ async addImageURLs(message, attachments) {
+ const { files, image_urls } = await encodeAndFormat(this.options.req, attachments);
+
+ message.image_urls = image_urls;
+ return files;
+ }
+
async buildMessages(
messages,
parentMessageId,
@@ -355,13 +458,23 @@ class OpenAIClient extends BaseClient {
}
if (this.options.attachments) {
- const attachments = await this.options.attachments;
- const { files, image_urls } = await encodeAndFormat(
- this.options.req,
- attachments.filter((file) => file.type.includes('image')),
+ const attachments = (await this.options.attachments).filter((file) =>
+ file.type.includes('image'),
+ );
+
+ if (this.message_file_map) {
+ this.message_file_map[orderedMessages[orderedMessages.length - 1].messageId] = attachments;
+ } else {
+ this.message_file_map = {
+ [orderedMessages[orderedMessages.length - 1].messageId]: attachments,
+ };
+ }
+
+ const files = await this.addImageURLs(
+ orderedMessages[orderedMessages.length - 1],
+ attachments,
);
- orderedMessages[orderedMessages.length - 1].image_urls = image_urls;
this.options.attachments = files;
}
@@ -372,10 +485,25 @@ class OpenAIClient extends BaseClient {
assistantName: this.options?.chatGptLabel,
});
- if (this.contextStrategy && !orderedMessages[i].tokenCount) {
+ const needsTokenCount = this.contextStrategy && !orderedMessages[i].tokenCount;
+
+ /* If tokens were never counted, or, is a Vision request and the message has files, count again */
+ if (needsTokenCount || (this.isVisionModel && (message.image_urls || message.files))) {
orderedMessages[i].tokenCount = this.getTokenCountForMessage(formattedMessage);
}
+ /* If message has files, calculate image token cost */
+ if (this.message_file_map && this.message_file_map[message.messageId]) {
+ const attachments = this.message_file_map[message.messageId];
+ for (const file of attachments) {
+ orderedMessages[i].tokenCount += this.calculateImageTokenCost({
+ width: file.width,
+ height: file.height,
+ detail: this.options.imageDetail ?? ImageDetail.auto,
+ });
+ }
+ }
+
return formattedMessage;
});
@@ -780,7 +908,6 @@ ${convo}
if (this.isChatCompletion) {
modelOptions.messages = payload;
} else {
- // TODO: unreachable code. Need to implement completions call for non-chat models
modelOptions.prompt = payload;
}
@@ -916,6 +1043,8 @@ ${convo}
clientOptions.addMetadata({ finish_reason });
}
+ logger.debug('[OpenAIClient] chatCompletion response', chatCompletion);
+
return message.content;
} catch (err) {
if (
diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js
index f26df8a2d1..6118c3547a 100644
--- a/api/app/clients/PluginsClient.js
+++ b/api/app/clients/PluginsClient.js
@@ -112,7 +112,7 @@ class PluginsClient extends OpenAIClient {
signal: this.abortController.signal,
openAIApiKey: this.openAIApiKey,
conversationId: this.conversationId,
- debug: this.options?.debug,
+ fileStrategy: this.options.req.app.locals.fileStrategy,
message,
},
});
diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js
index a31d82fe12..8c2226215c 100644
--- a/api/app/clients/specs/OpenAIClient.test.js
+++ b/api/app/clients/specs/OpenAIClient.test.js
@@ -546,6 +546,39 @@ describe('OpenAIClient', () => {
expect(totalTokens).toBe(testCase.expected);
});
});
+
+ const vision_request = [
+ {
+ role: 'user',
+ content: [
+ {
+ type: 'text',
+ text: 'describe what is in this image?',
+ },
+ {
+ type: 'image_url',
+ image_url: {
+ url: 'https://venturebeat.com/wp-content/uploads/2019/03/openai-1.png',
+ detail: 'high',
+ },
+ },
+ ],
+ },
+ ];
+
+ const expectedTokens = 14;
+ const visionModel = 'gpt-4-vision-preview';
+
+ it(`should return ${expectedTokens} tokens for model ${visionModel} (Vision Request)`, () => {
+ client.modelOptions.model = visionModel;
+ client.selectTokenizer();
+ // 3 tokens for assistant label
+ let totalTokens = 3;
+ for (let message of vision_request) {
+ totalTokens += client.getTokenCountForMessage(message);
+ }
+ expect(totalTokens).toBe(expectedTokens);
+ });
});
describe('sendMessage/getCompletion/chatCompletion', () => {
diff --git a/api/app/clients/tools/DALL-E.js b/api/app/clients/tools/DALL-E.js
index 387294a1cb..7f0c952030 100644
--- a/api/app/clients/tools/DALL-E.js
+++ b/api/app/clients/tools/DALL-E.js
@@ -1,20 +1,13 @@
// From https://platform.openai.com/docs/api-reference/images/create
// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
-const path = require('path');
const OpenAI = require('openai');
// const { genAzureEndpoint } = require('~/utils/genAzureEndpoints');
const { v4: uuidv4 } = require('uuid');
const { Tool } = require('langchain/tools');
const { HttpsProxyAgent } = require('https-proxy-agent');
-const {
- saveImageToFirebaseStorage,
- getFirebaseStorageImageUrl,
- getFirebaseStorage,
-} = require('~/server/services/Files/Firebase');
const { getImageBasename } = require('~/server/services/Files/images');
+const { processFileURL } = require('~/server/services/Files/process');
const extractBaseURL = require('~/utils/extractBaseURL');
-const saveImageFromUrl = require('./saveImageFromUrl');
const { logger } = require('~/config');
const { DALLE_REVERSE_PROXY, PROXY } = process.env;
@@ -23,6 +16,7 @@ class OpenAICreateImage extends Tool {
super();
this.userId = fields.userId;
+ this.fileStrategy = fields.fileStrategy;
let apiKey = fields.DALLE_API_KEY || this.getApiKey();
const config = { apiKey };
@@ -82,12 +76,8 @@ Guidelines:
.trim();
}
- getMarkdownImageUrl(imageName) {
- const imageUrl = path
- .join(this.relativeImageUrl, imageName)
- .replace(/\\/g, '/')
- .replace('public/', '');
- return `![generated image](/${imageUrl})`;
+ wrapInMarkdown(imageUrl) {
+ return `![generated image](${imageUrl})`;
}
async _call(input) {
@@ -118,45 +108,21 @@ Guidelines:
});
}
- this.outputPath = path.resolve(
- __dirname,
- '..',
- '..',
- '..',
- '..',
- 'client',
- 'public',
- 'images',
- this.userId,
- );
-
- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', 'client');
- this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
- // Check if directory exists, if not create it
- if (!fs.existsSync(this.outputPath)) {
- fs.mkdirSync(this.outputPath, { recursive: true });
- }
+ try {
+ const result = await processFileURL({
+ fileStrategy: this.fileStrategy,
+ userId: this.userId,
+ URL: theImageUrl,
+ fileName: imageName,
+ basePath: 'images',
+ });
- const storage = getFirebaseStorage();
- if (storage) {
- try {
- await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);
- this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);
- logger.debug('[DALL-E] result: ' + this.result);
- } catch (error) {
- logger.error('Error while saving the image to Firebase Storage:', error);
- this.result = `Failed to save the image to Firebase Storage. ${error.message}`;
- }
- } else {
- try {
- await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
- this.result = this.getMarkdownImageUrl(imageName);
- } catch (error) {
- logger.error('Error while saving the image locally:', error);
- this.result = `Failed to save the image locally. ${error.message}`;
- }
+ this.result = this.wrapInMarkdown(result);
+ } catch (error) {
+ logger.error('Error while saving the image:', error);
+ this.result = `Failed to save the image locally. ${error.message}`;
}
+
return this.result;
}
}
diff --git a/api/app/clients/tools/saveImageFromUrl.js b/api/app/clients/tools/saveImageFromUrl.js
deleted file mode 100644
index b9fbcc895b..0000000000
--- a/api/app/clients/tools/saveImageFromUrl.js
+++ /dev/null
@@ -1,46 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const axios = require('axios');
-const { logger } = require('~/config');
-
-async function saveImageFromUrl(url, outputPath, outputFilename) {
- try {
- // Fetch the image from the URL
- const response = await axios({
- url,
- responseType: 'stream',
- });
-
- // Get the content type from the response headers
- const contentType = response.headers['content-type'];
- let extension = contentType.split('/').pop();
-
- // Check if the output directory exists, if not, create it
- if (!fs.existsSync(outputPath)) {
- fs.mkdirSync(outputPath, { recursive: true });
- }
-
- // Replace or append the correct extension
- const extRegExp = new RegExp(path.extname(outputFilename) + '$');
- outputFilename = outputFilename.replace(extRegExp, `.${extension}`);
- if (!path.extname(outputFilename)) {
- outputFilename += `.${extension}`;
- }
-
- // Create a writable stream for the output path
- const outputFilePath = path.join(outputPath, outputFilename);
- const writer = fs.createWriteStream(outputFilePath);
-
- // Pipe the response data to the output file
- response.data.pipe(writer);
-
- return new Promise((resolve, reject) => {
- writer.on('finish', resolve);
- writer.on('error', reject);
- });
- } catch (error) {
- logger.error('[saveImageFromUrl] Error while saving the image:', error);
- }
-}
-
-module.exports = saveImageFromUrl;
diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js
index 17d0368f39..16326acdee 100644
--- a/api/app/clients/tools/structured/DALLE3.js
+++ b/api/app/clients/tools/structured/DALLE3.js
@@ -1,20 +1,13 @@
// From https://platform.openai.com/docs/guides/images/usage?context=node
// To use this tool, you must pass in a configured OpenAIApi object.
-const fs = require('fs');
-const path = require('path');
const { z } = require('zod');
const OpenAI = require('openai');
const { v4: uuidv4 } = require('uuid');
const { Tool } = require('langchain/tools');
const { HttpsProxyAgent } = require('https-proxy-agent');
-const {
- saveImageToFirebaseStorage,
- getFirebaseStorageImageUrl,
- getFirebaseStorage,
-} = require('~/server/services/Files/Firebase');
const { getImageBasename } = require('~/server/services/Files/images');
+const { processFileURL } = require('~/server/services/Files/process');
const extractBaseURL = require('~/utils/extractBaseURL');
-const saveImageFromUrl = require('../saveImageFromUrl');
const { logger } = require('~/config');
const { DALLE3_SYSTEM_PROMPT, DALLE_REVERSE_PROXY, PROXY } = process.env;
@@ -23,6 +16,7 @@ class DALLE3 extends Tool {
super();
this.userId = fields.userId;
+ this.fileStrategy = fields.fileStrategy;
let apiKey = fields.DALLE_API_KEY || this.getApiKey();
const config = { apiKey };
if (DALLE_REVERSE_PROXY) {
@@ -91,12 +85,8 @@ class DALLE3 extends Tool {
.trim();
}
- getMarkdownImageUrl(imageName) {
- const imageUrl = path
- .join(this.relativeImageUrl, imageName)
- .replace(/\\/g, '/')
- .replace('public/', '');
- return `![generated image](/${imageUrl})`;
+ wrapInMarkdown(imageUrl) {
+ return `![generated image](${imageUrl})`;
}
async _call(data) {
@@ -143,43 +133,19 @@ Error Message: ${error.message}`;
});
}
- this.outputPath = path.resolve(
- __dirname,
- '..',
- '..',
- '..',
- '..',
- '..',
- 'client',
- 'public',
- 'images',
- this.userId,
- );
- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
- this.relativeImageUrl = path.relative(appRoot, this.outputPath);
-
- // Check if directory exists, if not create it
- if (!fs.existsSync(this.outputPath)) {
- fs.mkdirSync(this.outputPath, { recursive: true });
- }
- const storage = getFirebaseStorage();
- if (storage) {
- try {
- await saveImageToFirebaseStorage(this.userId, theImageUrl, imageName);
- this.result = await getFirebaseStorageImageUrl(`${this.userId}/${imageName}`);
- logger.debug('[DALL-E-3] result: ' + this.result);
- } catch (error) {
- logger.error('Error while saving the image to Firebase Storage:', error);
- this.result = `Failed to save the image to Firebase Storage. ${error.message}`;
- }
- } else {
- try {
- await saveImageFromUrl(theImageUrl, this.outputPath, imageName);
- this.result = this.getMarkdownImageUrl(imageName);
- } catch (error) {
- logger.error('Error while saving the image locally:', error);
- this.result = `Failed to save the image locally. ${error.message}`;
- }
+ try {
+ const result = await processFileURL({
+ fileStrategy: this.fileStrategy,
+ userId: this.userId,
+ URL: theImageUrl,
+ fileName: imageName,
+ basePath: 'images',
+ });
+
+ this.result = this.wrapInMarkdown(result);
+ } catch (error) {
+ logger.error('Error while saving the image:', error);
+ this.result = `Failed to save the image locally. ${error.message}`;
}
return this.result;
diff --git a/api/app/clients/tools/structured/specs/DALLE3.spec.js b/api/app/clients/tools/structured/specs/DALLE3.spec.js
index 34fa3ebf00..65a5fbd844 100644
--- a/api/app/clients/tools/structured/specs/DALLE3.spec.js
+++ b/api/app/clients/tools/structured/specs/DALLE3.spec.js
@@ -1,20 +1,13 @@
-const fs = require('fs');
-const path = require('path');
const OpenAI = require('openai');
const DALLE3 = require('../DALLE3');
-const {
- getFirebaseStorage,
- saveImageToFirebaseStorage,
-} = require('~/server/services/Files/Firebase');
-const saveImageFromUrl = require('../../saveImageFromUrl');
+const { processFileURL } = require('~/server/services/Files/process');
+
const { logger } = require('~/config');
jest.mock('openai');
-jest.mock('~/server/services/Files/Firebase', () => ({
- getFirebaseStorage: jest.fn(),
- saveImageToFirebaseStorage: jest.fn(),
- getFirebaseStorageImageUrl: jest.fn(),
+jest.mock('~/server/services/Files/process', () => ({
+ processFileURL: jest.fn(),
}));
jest.mock('~/server/services/Files/images', () => ({
@@ -50,10 +43,6 @@ jest.mock('fs', () => {
};
});
-jest.mock('../../saveImageFromUrl', () => {
- return jest.fn();
-});
-
jest.mock('path', () => {
return {
resolve: jest.fn(),
@@ -99,10 +88,8 @@ describe('DALLE3', () => {
it('should generate markdown image URL correctly', () => {
const imageName = 'test.png';
- path.join.mockReturnValue('images/test.png');
- path.relative.mockReturnValue('images/test.png');
- const markdownImage = dalle.getMarkdownImageUrl(imageName);
- expect(markdownImage).toBe('![generated image](/images/test.png)');
+ const markdownImage = dalle.wrapInMarkdown(imageName);
+ expect(markdownImage).toBe('![generated image](test.png)');
});
it('should call OpenAI API with correct parameters', async () => {
@@ -122,11 +109,7 @@ describe('DALLE3', () => {
};
generate.mockResolvedValue(mockResponse);
- saveImageFromUrl.mockResolvedValue(true);
- fs.existsSync.mockReturnValue(true);
- path.resolve.mockReturnValue('/fakepath/images');
- path.join.mockReturnValue('/fakepath/images/img-test.png');
- path.relative.mockReturnValue('images/img-test.png');
+ processFileURL.mockResolvedValue('http://example.com/img-test.png');
const result = await dalle._call(mockData);
@@ -138,6 +121,7 @@ describe('DALLE3', () => {
prompt: mockData.prompt,
n: 1,
});
+
expect(result).toContain('![generated image]');
});
@@ -184,23 +168,6 @@ describe('DALLE3', () => {
});
});
- it('should create the directory if it does not exist', async () => {
- const mockData = {
- prompt: 'A test prompt',
- };
- const mockResponse = {
- data: [
- {
- url: 'http://example.com/img-test.png',
- },
- ],
- };
- generate.mockResolvedValue(mockResponse);
- fs.existsSync.mockReturnValue(false); // Simulate directory does not exist
- await dalle._call(mockData);
- expect(fs.mkdirSync).toHaveBeenCalledWith(expect.any(String), { recursive: true });
- });
-
it('should log an error and return the image URL if there is an error saving the image', async () => {
const mockData = {
prompt: 'A test prompt',
@@ -214,31 +181,12 @@ describe('DALLE3', () => {
};
const error = new Error('Error while saving the image');
generate.mockResolvedValue(mockResponse);
- saveImageFromUrl.mockRejectedValue(error);
+ processFileURL.mockRejectedValue(error);
const result = await dalle._call(mockData);
- expect(logger.error).toHaveBeenCalledWith('Error while saving the image locally:', error);
+ expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
expect(result).toBe('Failed to save the image locally. Error while saving the image');
});
- it('should save image to Firebase Storage if Firebase is initialized', async () => {
- const mockData = {
- prompt: 'A test prompt',
- };
- const mockImageUrl = 'http://example.com/img-test.png';
- const mockResponse = { data: [{ url: mockImageUrl }] };
- generate.mockResolvedValue(mockResponse);
- getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
-
- await dalle._call(mockData);
-
- expect(getFirebaseStorage).toHaveBeenCalled();
- expect(saveImageToFirebaseStorage).toHaveBeenCalledWith(
- undefined,
- mockImageUrl,
- expect.any(String),
- );
- });
-
it('should handle error when saving image to Firebase Storage fails', async () => {
const mockData = {
prompt: 'A test prompt',
@@ -247,17 +195,11 @@ describe('DALLE3', () => {
const mockResponse = { data: [{ url: mockImageUrl }] };
const error = new Error('Error while saving to Firebase');
generate.mockResolvedValue(mockResponse);
- getFirebaseStorage.mockReturnValue({}); // Simulate Firebase being initialized
- saveImageToFirebaseStorage.mockRejectedValue(error);
+ processFileURL.mockRejectedValue(error);
const result = await dalle._call(mockData);
- expect(logger.error).toHaveBeenCalledWith(
- 'Error while saving the image to Firebase Storage:',
- error,
- );
- expect(result).toBe(
- 'Failed to save the image to Firebase Storage. Error while saving to Firebase',
- );
+ expect(logger.error).toHaveBeenCalledWith('Error while saving the image:', error);
+ expect(result).toContain('Failed to save the image');
});
});
diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js
index 352dd5dec7..1d9a3a0074 100644
--- a/api/app/clients/tools/util/handleTools.js
+++ b/api/app/clients/tools/util/handleTools.js
@@ -170,6 +170,8 @@ const loadTools = async ({
const toolOptions = {
serpapi: { location: 'Austin,Texas,United States', hl: 'en', gl: 'us' },
+ dalle: { fileStrategy: options.fileStrategy },
+ 'dall-e': { fileStrategy: options.fileStrategy },
};
const toolAuthFields = {};
diff --git a/api/cache/keyvRedis.js b/api/cache/keyvRedis.js
index f723429ee2..9501045e4e 100644
--- a/api/cache/keyvRedis.js
+++ b/api/cache/keyvRedis.js
@@ -10,10 +10,11 @@ if (REDIS_URI && isEnabled(USE_REDIS)) {
keyvRedis = new KeyvRedis(REDIS_URI, { useRedisSets: false });
keyvRedis.on('error', (err) => logger.error('KeyvRedis connection error:', err));
keyvRedis.setMaxListeners(20);
-} else {
logger.info(
- '`REDIS_URI` not provided, or `USE_REDIS` not set. Redis module will not be initialized.',
+ '[Optional] Redis initialized. Note: Redis support is experimental. If you have issues, disable it. Cache needs to be flushed for values to refresh.',
);
+} else {
+ logger.info('[Optional] Redis not initialized. Note: Redis support is experimental.');
}
module.exports = keyvRedis;
diff --git a/api/config/paths.js b/api/config/paths.js
index 2f577a183f..41e3ac5054 100644
--- a/api/config/paths.js
+++ b/api/config/paths.js
@@ -1,6 +1,7 @@
const path = require('path');
module.exports = {
+ dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
};
diff --git a/api/models/schema/defaults.js b/api/models/schema/defaults.js
index feedf1019a..39a6430f46 100644
--- a/api/models/schema/defaults.js
+++ b/api/models/schema/defaults.js
@@ -84,6 +84,12 @@ const conversationPreset = {
type: String,
// default: null,
},
+ resendImages: {
+ type: Boolean,
+ },
+ imageDetail: {
+ type: String,
+ },
};
const agentOptions = {
diff --git a/api/models/schema/fileSchema.js b/api/models/schema/fileSchema.js
index bf9db4864d..471b7bfd70 100644
--- a/api/models/schema/fileSchema.js
+++ b/api/models/schema/fileSchema.js
@@ -1,3 +1,4 @@
+const { FileSources } = require('librechat-data-provider');
const mongoose = require('mongoose');
/**
@@ -12,6 +13,7 @@ const mongoose = require('mongoose');
* @property {'file'} object - Type of object, always 'file'
* @property {string} type - Type of file
* @property {number} usage - Number of uses of the file
+ * @property {string} [source] - The source of the file
* @property {number} [width] - Optional width of the file
* @property {number} [height] - Optional height of the file
* @property {Date} [expiresAt] - Optional height of the file
@@ -42,11 +44,6 @@ const fileSchema = mongoose.Schema(
type: Number,
required: true,
},
- usage: {
- type: Number,
- required: true,
- default: 0,
- },
filename: {
type: String,
required: true,
@@ -64,6 +61,15 @@ const fileSchema = mongoose.Schema(
type: String,
required: true,
},
+ usage: {
+ type: Number,
+ required: true,
+ default: 0,
+ },
+ source: {
+ type: String,
+ default: FileSources.local,
+ },
width: Number,
height: Number,
expiresAt: {
diff --git a/api/server/index.js b/api/server/index.js
index 3c6429e47d..86806b5914 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -5,33 +5,28 @@ const cors = require('cors');
const express = require('express');
const passport = require('passport');
const mongoSanitize = require('express-mongo-sanitize');
-const { initializeFirebase } = require('~/server/services/Files/Firebase/initialize');
-const loadCustomConfig = require('~/server/services/Config/loadCustomConfig');
-const errorController = require('~/server/controllers/ErrorController');
-const configureSocialLogins = require('~/server/socialLogins');
-const noIndex = require('~/server/middleware/noIndex');
+const errorController = require('./controllers/ErrorController');
+const { jwtLogin, passportLogin } = require('~/strategies');
+const configureSocialLogins = require('./socialLogins');
const { connectDb, indexSync } = require('~/lib/db');
+const AppService = require('./services/AppService');
+const noIndex = require('./middleware/noIndex');
const { logger } = require('~/config');
-const routes = require('~/server/routes');
-const paths = require('~/config/paths');
+const routes = require('./routes');
const { PORT, HOST, ALLOW_SOCIAL_LOGIN } = process.env ?? {};
const port = Number(PORT) || 3080;
const host = HOST || 'localhost';
-const projectPath = path.join(__dirname, '..', '..', 'client');
-const { jwtLogin, passportLogin } = require('~/strategies');
const startServer = async () => {
await connectDb();
logger.info('Connected to MongoDB');
- await loadCustomConfig();
- initializeFirebase();
await indexSync();
const app = express();
- app.locals.config = paths;
+ await AppService(app);
// Middleware
app.use(noIndex);
@@ -39,14 +34,14 @@ const startServer = async () => {
app.use(express.json({ limit: '3mb' }));
app.use(mongoSanitize());
app.use(express.urlencoded({ extended: true, limit: '3mb' }));
- app.use(express.static(path.join(projectPath, 'dist')));
- app.use(express.static(path.join(projectPath, 'public')));
+ app.use(express.static(app.locals.paths.dist));
+ app.use(express.static(app.locals.paths.publicPath));
app.set('trust proxy', 1); // trust first proxy
app.use(cors());
if (!ALLOW_SOCIAL_LOGIN) {
console.warn(
- 'Social logins are disabled. Set Envrionment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',
+ 'Social logins are disabled. Set Environment Variable "ALLOW_SOCIAL_LOGIN" to true to enable them.',
);
}
@@ -81,7 +76,7 @@ const startServer = async () => {
app.use('/api/files', routes.files);
app.use((req, res) => {
- res.status(404).sendFile(path.join(projectPath, 'dist', 'index.html'));
+ res.status(404).sendFile(path.join(app.locals.paths.dist, 'index.html'));
});
app.listen(port, host, () => {
diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.js
index 543815e367..91d0caceaa 100644
--- a/api/server/middleware/buildEndpointOption.js
+++ b/api/server/middleware/buildEndpointOption.js
@@ -1,10 +1,10 @@
-const { processFiles } = require('~/server/services/Files');
+const { parseConvo, EModelEndpoint } = require('librechat-data-provider');
+const { processFiles } = require('~/server/services/Files/process');
+const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
+const anthropic = require('~/server/services/Endpoints/anthropic');
const openAI = require('~/server/services/Endpoints/openAI');
const custom = require('~/server/services/Endpoints/custom');
const google = require('~/server/services/Endpoints/google');
-const anthropic = require('~/server/services/Endpoints/anthropic');
-const gptPlugins = require('~/server/services/Endpoints/gptPlugins');
-const { parseConvo, EModelEndpoint } = require('librechat-data-provider');
const buildFunction = {
[EModelEndpoint.openAI]: openAI.buildOptions,
diff --git a/api/server/routes/files/avatar.js b/api/server/routes/files/avatar.js
index a7bb07c0f9..5abba85f9e 100644
--- a/api/server/routes/files/avatar.js
+++ b/api/server/routes/files/avatar.js
@@ -1,7 +1,7 @@
const express = require('express');
const multer = require('multer');
-const uploadAvatar = require('~/server/services/Files/images/avatar/uploadAvatar');
+const uploadAvatar = require('~/server/services/Files/images/avatar');
const { requireJwtAuth } = require('~/server/middleware/');
const User = require('~/models/User');
@@ -23,7 +23,12 @@ router.post('/', requireJwtAuth, upload.single('input'), async (req, res) => {
if (!user) {
throw new Error('User not found');
}
- const url = await uploadAvatar(userId, input, manual);
+ const url = await uploadAvatar({
+ input,
+ userId,
+ manual,
+ fileStrategy: req.app.locals.fileStrategy,
+ });
res.json({ url });
} catch (error) {
diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js
index d9df1bdd75..3fea2e5d07 100644
--- a/api/server/routes/files/files.js
+++ b/api/server/routes/files/files.js
@@ -1,36 +1,29 @@
const { z } = require('zod');
-const path = require('path');
-const fs = require('fs').promises;
const express = require('express');
-const { deleteFiles } = require('~/models');
+const { FileSources } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { deleteFiles, getFiles } = require('~/models');
const { logger } = require('~/config');
const router = express.Router();
const isUUID = z.string().uuid();
-const isValidPath = (req, base, subfolder, filepath) => {
- const normalizedBase = path.resolve(base, subfolder, req.user.id);
- const normalizedFilepath = path.resolve(filepath);
- return normalizedFilepath.startsWith(normalizedBase);
-};
-
-const deleteFile = async (req, file) => {
- const { publicPath } = req.app.locals.config;
- const parts = file.filepath.split(path.sep);
- const subfolder = parts[1];
- const filepath = path.join(publicPath, file.filepath);
-
- if (!isValidPath(req, publicPath, subfolder, filepath)) {
- throw new Error('Invalid file path');
+router.get('/', async (req, res) => {
+ try {
+ const files = await getFiles({ user: req.user.id });
+ res.status(200).send(files);
+ } catch (error) {
+ logger.error('[/files] Error getting files:', error);
+ res.status(400).json({ message: 'Error in request', error: error.message });
}
-
- await fs.unlink(filepath);
-};
+});
router.delete('/', async (req, res) => {
try {
const { files: _files } = req.body;
+
+ /** @type {MongoFile[]} */
const files = _files.filter((file) => {
if (!file.file_id) {
return false;
@@ -47,9 +40,24 @@ router.delete('/', async (req, res) => {
}
const file_ids = files.map((file) => file.file_id);
+ const deletionMethods = {};
const promises = [];
promises.push(await deleteFiles(file_ids));
+
for (const file of files) {
+ const source = file.source ?? FileSources.local;
+
+ if (deletionMethods[source]) {
+ promises.push(deletionMethods[source](req, file));
+ continue;
+ }
+
+ const { deleteFile } = getStrategyFunctions(source);
+ if (!deleteFile) {
+ throw new Error(`Delete function not implemented for ${source}`);
+ }
+
+ deletionMethods[source] = deleteFile;
promises.push(deleteFile(req, file));
}
diff --git a/api/server/routes/files/images.js b/api/server/routes/files/images.js
index f88b7f2c7a..30d3c3cac6 100644
--- a/api/server/routes/files/images.js
+++ b/api/server/routes/files/images.js
@@ -2,7 +2,7 @@ const { z } = require('zod');
const fs = require('fs').promises;
const express = require('express');
const upload = require('./multer');
-const { localStrategy } = require('~/server/services/Files');
+const { processImageUpload } = require('~/server/services/Files/process');
const { logger } = require('~/config');
const router = express.Router();
@@ -34,7 +34,8 @@ router.post('/', upload.single('file'), async (req, res) => {
uuidSchema.parse(metadata.file_id);
metadata.temp_file_id = metadata.file_id;
metadata.file_id = req.file_id;
- await localStrategy({ req, res, file, metadata });
+
+ await processImageUpload({ req, res, file, metadata });
} catch (error) {
logger.error('[/files/images] Error processing file:', error);
try {
diff --git a/api/server/routes/files/index.js b/api/server/routes/files/index.js
index 74b200c806..9afb900bbe 100644
--- a/api/server/routes/files/index.js
+++ b/api/server/routes/files/index.js
@@ -11,6 +11,7 @@ const {
const files = require('./files');
const images = require('./images');
+const avatar = require('./avatar');
router.use(requireJwtAuth);
router.use(checkBan);
@@ -18,6 +19,6 @@ router.use(uaParser);
router.use('/', files);
router.use('/images', images);
-router.use('/images/avatar', require('./avatar'));
+router.use('/images/avatar', avatar);
module.exports = router;
diff --git a/api/server/routes/files/multer.js b/api/server/routes/files/multer.js
index ae0158f7d5..d5aea05a37 100644
--- a/api/server/routes/files/multer.js
+++ b/api/server/routes/files/multer.js
@@ -8,7 +8,7 @@ const sizeLimit = 20 * 1024 * 1024; // 20 MB
const storage = multer.diskStorage({
destination: function (req, file, cb) {
- const outputPath = path.join(req.app.locals.config.imageOutput, 'temp');
+ const outputPath = path.join(req.app.locals.paths.imageOutput, 'temp');
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath, { recursive: true });
}
diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js
new file mode 100644
index 0000000000..b1f7cf57d8
--- /dev/null
+++ b/api/server/services/AppService.js
@@ -0,0 +1,27 @@
+const { FileSources } = require('librechat-data-provider');
+const { initializeFirebase } = require('./Files/Firebase/initialize');
+const loadCustomConfig = require('./Config/loadCustomConfig');
+const paths = require('~/config/paths');
+
+/**
+ *
+ * Loads custom config and initializes app-wide variables.
+ * @function AppService
+ * @param {Express.Application} app - The Express application object.
+ */
+const AppService = async (app) => {
+ const config = (await loadCustomConfig()) ?? {};
+ const fileStrategy = config.fileStrategy ?? FileSources.local;
+ process.env.CDN_PROVIDER = fileStrategy;
+
+ if (fileStrategy === FileSources.firebase) {
+ initializeFirebase();
+ }
+
+ app.locals = {
+ fileStrategy,
+ paths,
+ };
+};
+
+module.exports = AppService;
diff --git a/api/server/services/Endpoints/openAI/buildOptions.js b/api/server/services/Endpoints/openAI/buildOptions.js
index a1ad232bb7..80037fb4b8 100644
--- a/api/server/services/Endpoints/openAI/buildOptions.js
+++ b/api/server/services/Endpoints/openAI/buildOptions.js
@@ -1,9 +1,11 @@
const buildOptions = (endpoint, parsedBody) => {
- const { chatGptLabel, promptPrefix, ...rest } = parsedBody;
+ const { chatGptLabel, promptPrefix, resendImages, imageDetail, ...rest } = parsedBody;
const endpointOption = {
endpoint,
chatGptLabel,
promptPrefix,
+ resendImages,
+ imageDetail,
modelOptions: {
...rest,
},
diff --git a/api/server/services/Files/Firebase/crud.js b/api/server/services/Files/Firebase/crud.js
new file mode 100644
index 0000000000..68f534bcb6
--- /dev/null
+++ b/api/server/services/Files/Firebase/crud.js
@@ -0,0 +1,174 @@
+const fetch = require('node-fetch');
+const { ref, uploadBytes, getDownloadURL, deleteObject } = require('firebase/storage');
+const { getFirebaseStorage } = require('./initialize');
+
+/**
+ * Deletes a file from Firebase Storage.
+ * @param {string} directory - The directory name
+ * @param {string} fileName - The name of the file to delete.
+ * @returns {Promise<void>} A promise that resolves when the file is deleted.
+ */
+async function deleteFile(basePath, fileName) {
+ const storage = getFirebaseStorage();
+ if (!storage) {
+ console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
+ throw new Error('Firebase is not initialized');
+ }
+
+ const storageRef = ref(storage, `${basePath}/${fileName}`);
+
+ try {
+ await deleteObject(storageRef);
+ console.log('File deleted successfully from Firebase Storage');
+ } catch (error) {
+ console.error('Error deleting file from Firebase Storage:', error.message);
+ throw error;
+ }
+}
+
+/**
+ * Saves an file from a given URL to Firebase Storage. The function first initializes the Firebase Storage
+ * reference, then uploads the file to a specified basePath in the Firebase Storage. It handles initialization
+ * errors and upload errors, logging them to the console. If the upload is successful, the file name is returned.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
+ * in Firebase Storage.
+ * @param {string} params.URL - The URL of the file to be uploaded. The file at this URL will be fetched
+ * and uploaded to Firebase Storage.
+ * @param {string} params.fileName - The name that will be used to save the file in Firebase Storage. This
+ * should include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
+ * be stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ * A promise that resolves to the file name if the file is successfully uploaded, or null if there
+ * is an error in initialization or upload.
+ */
+async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {
+ const storage = getFirebaseStorage();
+ if (!storage) {
+ console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
+ return null;
+ }
+
+ const storageRef = ref(storage, `${basePath}/${userId.toString()}/${fileName}`);
+
+ try {
+ await uploadBytes(storageRef, await fetch(URL).then((response) => response.buffer()));
+ return fileName;
+ } catch (error) {
+ console.error('Error uploading file to Firebase Storage:', error.message);
+ return null;
+ }
+}
+
+/**
+ * Retrieves the download URL for a specified file from Firebase Storage. This function initializes the
+ * Firebase Storage and generates a reference to the file based on the provided basePath and file name. If
+ * Firebase Storage is not initialized or if there is an error in fetching the URL, the error is logged
+ * to the console.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.fileName - The name of the file for which the URL is to be retrieved. This should
+ * include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file is
+ * stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ * A promise that resolves to the download URL of the file if successful, or null if there is an
+ * error in initialization or fetching the URL.
+ */
+async function getFirebaseURL({ fileName, basePath = 'images' }) {
+ const storage = getFirebaseStorage();
+ if (!storage) {
+ console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
+ return null;
+ }
+
+ const storageRef = ref(storage, `${basePath}/${fileName}`);
+
+ try {
+ return await getDownloadURL(storageRef);
+ } catch (error) {
+ console.error('Error fetching file URL from Firebase Storage:', error.message);
+ return null;
+ }
+}
+
+/**
+ * Uploads a buffer to Firebase Storage.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
+ * in Firebase Storage.
+ * @param {string} params.fileName - The name of the file to be saved in Firebase Storage.
+ * @param {string} params.buffer - The buffer to be uploaded.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
+ * be stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string>} - A promise that resolves to the download URL of the uploaded file.
+ */
+async function saveBufferToFirebase({ userId, buffer, fileName, basePath = 'images' }) {
+ const storage = getFirebaseStorage();
+ if (!storage) {
+ throw new Error('Firebase is not initialized');
+ }
+
+ const storageRef = ref(storage, `${basePath}/${userId}/${fileName}`);
+ await uploadBytes(storageRef, buffer);
+
+ // Assuming you have a function to get the download URL
+ return await getFirebaseURL({ fileName, basePath: `${basePath}/${userId}` });
+}
+
+/**
+ * Extracts and decodes the file path from a Firebase Storage URL.
+ *
+ * @param {string} urlString - The Firebase Storage URL.
+ * @returns {string} The decoded file path.
+ */
+function extractFirebaseFilePath(urlString) {
+ try {
+ const url = new URL(urlString);
+ const pathRegex = /\/o\/(.+?)(\?|$)/;
+ const match = url.pathname.match(pathRegex);
+
+ if (match && match[1]) {
+ return decodeURIComponent(match[1]);
+ }
+
+ return '';
+ } catch (error) {
+ // If URL parsing fails, return an empty string
+ return '';
+ }
+}
+
+/**
+ * Deletes a file from Firebase storage. This function determines the filepath from the
+ * Firebase storage URL via regex for deletion. Validated by the user's ID.
+ *
+ * @param {Express.Request} req - The request object from Express.
+ * It should contain a `user` object with an `id` property.
+ * @param {MongoFile} file - The file object to be deleted.
+ *
+ * @returns {Promise<void>}
+ * A promise that resolves when the file has been successfully deleted from Firebase storage.
+ * Throws an error if there is an issue with deletion.
+ */
+const deleteFirebaseFile = async (req, file) => {
+ const fileName = extractFirebaseFilePath(file.filepath);
+ if (!fileName.includes(req.user.id)) {
+ throw new Error('Invalid file path');
+ }
+ await deleteFile('', fileName);
+};
+
+module.exports = {
+ deleteFile,
+ getFirebaseURL,
+ saveURLToFirebase,
+ deleteFirebaseFile,
+ saveBufferToFirebase,
+};
diff --git a/api/server/services/Files/Firebase/images.js b/api/server/services/Files/Firebase/images.js
index e04902c02f..95b600962f 100644
--- a/api/server/services/Files/Firebase/images.js
+++ b/api/server/services/Files/Firebase/images.js
@@ -1,45 +1,105 @@
-const fetch = require('node-fetch');
-const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');
-const { getFirebaseStorage } = require('./initialize');
-
-async function saveImageToFirebaseStorage(userId, imageUrl, imageName) {
- const storage = getFirebaseStorage();
- if (!storage) {
- console.error('Firebase is not initialized. Cannot save image to Firebase Storage.');
- return null;
- }
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { resizeImage } = require('../images/resize');
+const { saveBufferToFirebase } = require('./crud');
+const { updateFile } = require('~/models/File');
+const { logger } = require('~/config');
- const storageRef = ref(storage, `images/${userId.toString()}/${imageName}`);
+/**
+ * Converts an image file to the WebP format. The function first resizes the image based on the specified
+ * resolution.
+ *
+ *
+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`
+ * representing the user, and an `app.locals.paths` object with an `imageOutput` path.
+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should
+ * have a `path` property that points to the location of the uploaded file.
+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}
+ * A promise that resolves to an object containing:
+ * - filepath: The path where the converted WebP image is saved.
+ * - bytes: The size of the converted image in bytes.
+ * - width: The width of the converted image.
+ * - height: The height of the converted image.
+ */
+async function uploadImageToFirebase(req, file, resolution = 'high') {
+ const inputFilePath = file.path;
+ const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+ const extension = path.extname(inputFilePath);
+ const userId = req.user.id;
- try {
- // Upload image to Firebase Storage using the image URL
- await uploadBytes(storageRef, await fetch(imageUrl).then((response) => response.buffer()));
- return imageName;
- } catch (error) {
- console.error('Error uploading image to Firebase Storage:', error.message);
- return null;
+ let webPBuffer;
+ let fileName = path.basename(inputFilePath);
+ if (extension.toLowerCase() === '.webp') {
+ webPBuffer = resizedBuffer;
+ } else {
+ webPBuffer = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+ // Replace or append the correct extension
+ const extRegExp = new RegExp(path.extname(fileName) + '$');
+ fileName = fileName.replace(extRegExp, '.webp');
+ if (!path.extname(fileName)) {
+ fileName += '.webp';
+ }
}
-}
-async function getFirebaseStorageImageUrl(imageName) {
- const storage = getFirebaseStorage();
- if (!storage) {
- console.error('Firebase is not initialized. Cannot get image URL from Firebase Storage.');
- return null;
- }
+ const downloadURL = await saveBufferToFirebase({ userId, buffer: webPBuffer, fileName });
+
+ await fs.promises.unlink(inputFilePath);
- const storageRef = ref(storage, `images/${imageName}`);
+ const bytes = Buffer.byteLength(webPBuffer);
+ return { filepath: downloadURL, bytes, width, height };
+}
+/**
+ * Local: Updates the file and returns the URL in expected order/format
+ * for image payload handling: tuple order of [filepath, URL].
+ * @param {Object} req - The request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
+ */
+async function prepareImageURL(req, file) {
+ const { filepath } = file;
+ const promises = [];
+ promises.push(updateFile({ file_id: file.file_id }));
+ promises.push(filepath);
+ return await Promise.all(promises);
+}
+
+/**
+ * Uploads a user's avatar to Firebase Storage and returns the URL.
+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.
+ *
+ * @param {object} params - The parameters object.
+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.
+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`
+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').
+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.
+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.
+ */
+async function processFirebaseAvatar({ buffer, User, manual }) {
try {
- // Get the download URL for the image from Firebase Storage
- return `![generated image](${await getDownloadURL(storageRef)})`;
+ const downloadURL = await saveBufferToFirebase({
+ userId: User._id.toString(),
+ buffer,
+ fileName: 'avatar.png',
+ });
+
+ const isManual = manual === 'true';
+
+ const url = `${downloadURL}?manual=${isManual}`;
+
+ if (isManual) {
+ User.avatar = url;
+ await User.save();
+ }
+
+ return url;
} catch (error) {
- console.error('Error fetching image URL from Firebase Storage:', error.message);
- return null;
+ logger.error('Error uploading profile picture:', error);
+ throw error;
}
}
-module.exports = {
- saveImageToFirebaseStorage,
- getFirebaseStorageImageUrl,
-};
+module.exports = { uploadImageToFirebase, prepareImageURL, processFirebaseAvatar };
diff --git a/api/server/services/Files/Firebase/index.js b/api/server/services/Files/Firebase/index.js
index 905bf660d4..27ad97a852 100644
--- a/api/server/services/Files/Firebase/index.js
+++ b/api/server/services/Files/Firebase/index.js
@@ -1,7 +1,9 @@
+const crud = require('./crud');
const images = require('./images');
const initialize = require('./initialize');
module.exports = {
+ ...crud,
...images,
...initialize,
};
diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.js
new file mode 100644
index 0000000000..d81c063031
--- /dev/null
+++ b/api/server/services/Files/Local/crud.js
@@ -0,0 +1,174 @@
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+const { logger } = require('~/config');
+const paths = require('~/config/paths');
+
+/**
+ * Saves a file to a specified output path with a new filename.
+ *
+ * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.
+ * @param {string} outputPath - The path where the file should be saved.
+ * @param {string} outputFilename - The new filename for the saved file (without extension).
+ * @returns {Promise<string>} The full path of the saved file.
+ * @throws Will throw an error if the file saving process fails.
+ */
+async function saveFile(file, outputPath, outputFilename) {
+ try {
+ if (!fs.existsSync(outputPath)) {
+ fs.mkdirSync(outputPath, { recursive: true });
+ }
+
+ const fileExtension = path.extname(file.originalname);
+ const filenameWithExt = outputFilename + fileExtension;
+ const outputFilePath = path.join(outputPath, filenameWithExt);
+ fs.copyFileSync(file.path, outputFilePath);
+ fs.unlinkSync(file.path);
+
+ return outputFilePath;
+ } catch (error) {
+ logger.error('[saveFile] Error while saving the file:', error);
+ throw error;
+ }
+}
+
+/**
+ * Saves an uploaded image file to a specified directory based on the user's ID and a filename.
+ *
+ * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.
+ * @param {Express.Multer.File} file - The uploaded file object.
+ * @param {string} filename - The new filename to assign to the saved image (without extension).
+ * @returns {Promise<void>}
+ * @throws Will throw an error if the image saving process fails.
+ */
+const saveLocalImage = async (req, file, filename) => {
+ const imagePath = req.app.locals.paths.imageOutput;
+ const outputPath = path.join(imagePath, req.user.id ?? '');
+ await saveFile(file, outputPath, filename);
+};
+
+/**
+ * Saves a file from a given URL to a local directory. The function fetches the file using the provided URL,
+ * determines the content type, and saves it to a specified local directory with the correct file extension.
+ * If the specified directory does not exist, it is created. The function returns the name of the saved file
+ * or null in case of an error.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific path
+ * in the local file system.
+ * @param {string} params.URL - The URL of the file to be downloaded and saved.
+ * @param {string} params.fileName - The desired file name for the saved file. This may be modified to include
+ * the correct file extension based on the content type.
+ * @param {string} [params.basePath='images'] - Optional. The base directory where the file will be saved.
+ * Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string|null>}
+ * A promise that resolves to the file name if the file is successfully saved, or null if there is an error.
+ */
+async function saveFileFromURL({ userId, URL, fileName, basePath = 'images' }) {
+ try {
+ // Fetch the file from the URL
+ const response = await axios({
+ url: URL,
+ responseType: 'stream',
+ });
+
+ // Get the content type from the response headers
+ const contentType = response.headers['content-type'];
+ let extension = contentType.split('/').pop();
+
+ // Construct the outputPath based on the basePath and userId
+ const outputPath = path.join(paths.publicPath, basePath, userId.toString());
+
+ // Check if the output directory exists, if not, create it
+ if (!fs.existsSync(outputPath)) {
+ fs.mkdirSync(outputPath, { recursive: true });
+ }
+
+ // Replace or append the correct extension
+ const extRegExp = new RegExp(path.extname(fileName) + '$');
+ fileName = fileName.replace(extRegExp, `.${extension}`);
+ if (!path.extname(fileName)) {
+ fileName += `.${extension}`;
+ }
+
+ // Create a writable stream for the output path
+ const outputFilePath = path.join(outputPath, fileName);
+ const writer = fs.createWriteStream(outputFilePath);
+
+ // Pipe the response data to the output file
+ response.data.pipe(writer);
+
+ return new Promise((resolve, reject) => {
+ writer.on('finish', () => resolve(fileName));
+ writer.on('error', reject);
+ });
+ } catch (error) {
+ logger.error('[saveFileFromURL] Error while saving the file:', error);
+ return null;
+ }
+}
+
+/**
+ * Constructs a local file path for a given file name and base path. This function simply joins the base
+ * path and the file name to create a file path. It does not check for the existence of the file at the path.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.fileName - The name of the file for which the path is to be constructed. This should
+ * include the file extension.
+ * @param {string} [params.basePath='images'] - Optional. The base directory to be used for constructing the file path.
+ * Defaults to 'images' if not specified.
+ *
+ * @returns {string}
+ * The constructed local file path.
+ */
+async function getLocalFileURL({ fileName, basePath = 'images' }) {
+ return path.posix.join('/', basePath, fileName);
+}
+
+/**
+ * Validates if a given filepath is within a specified subdirectory under a base path. This function constructs
+ * the expected base path using the base, subfolder, and user id from the request, and then checks if the
+ * provided filepath starts with this constructed base path.
+ *
+ * @param {Express.Request} req - The request object from Express. It should contain a `user` property with an `id`.
+ * @param {string} base - The base directory path.
+ * @param {string} subfolder - The subdirectory under the base path.
+ * @param {string} filepath - The complete file path to be validated.
+ *
+ * @returns {boolean}
+ * Returns true if the filepath is within the specified base and subfolder, false otherwise.
+ */
+const isValidPath = (req, base, subfolder, filepath) => {
+ const normalizedBase = path.resolve(base, subfolder, req.user.id);
+ const normalizedFilepath = path.resolve(filepath);
+ return normalizedFilepath.startsWith(normalizedBase);
+};
+
+/**
+ * Deletes a file from the filesystem. This function takes a file object, constructs the full path, and
+ * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.
+ *
+ * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with
+ * a `publicPath` property.
+ * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is
+ * a string representing the path of the file relative to the publicPath.
+ *
+ * @returns {Promise<void>}
+ * A promise that resolves when the file has been successfully deleted, or throws an error if the
+ * file path is invalid or if there is an error in deletion.
+ */
+const deleteLocalFile = async (req, file) => {
+ const { publicPath } = req.app.locals.paths;
+ const parts = file.filepath.split(path.sep);
+ const subfolder = parts[1];
+ const filepath = path.join(publicPath, file.filepath);
+
+ if (!isValidPath(req, publicPath, subfolder, filepath)) {
+ throw new Error('Invalid file path');
+ }
+
+ await fs.promises.unlink(filepath);
+};
+
+module.exports = { saveFile, saveLocalImage, saveFileFromURL, getLocalFileURL, deleteLocalFile };
diff --git a/api/server/services/Files/Local/images.js b/api/server/services/Files/Local/images.js
new file mode 100644
index 0000000000..63ed5b2f64
--- /dev/null
+++ b/api/server/services/Files/Local/images.js
@@ -0,0 +1,140 @@
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { resizeImage } = require('../images/resize');
+const { updateFile } = require('~/models/File');
+
+/**
+ * Converts an image file to the WebP format. The function first resizes the image based on the specified
+ * resolution.
+ *
+ * If the original image is already in WebP format, it writes the resized image back. Otherwise,
+ * it converts the image to WebP format before saving.
+ *
+ * The original image is deleted after conversion.
+ *
+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`
+ * representing the user, and an `app.locals.paths` object with an `imageOutput` path.
+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should
+ * have a `path` property that points to the location of the uploaded file.
+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}
+ * A promise that resolves to an object containing:
+ * - filepath: The path where the converted WebP image is saved.
+ * - bytes: The size of the converted image in bytes.
+ * - width: The width of the converted image.
+ * - height: The height of the converted image.
+ */
+async function uploadLocalImage(req, file, resolution = 'high') {
+ const inputFilePath = file.path;
+ const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+ const extension = path.extname(inputFilePath);
+
+ const { imageOutput } = req.app.locals.paths;
+ const userPath = path.join(imageOutput, req.user.id);
+
+ if (!fs.existsSync(userPath)) {
+ fs.mkdirSync(userPath, { recursive: true });
+ }
+
+ const newPath = path.join(userPath, path.basename(inputFilePath));
+
+ if (extension.toLowerCase() === '.webp') {
+ const bytes = Buffer.byteLength(resizedBuffer);
+ await fs.promises.writeFile(newPath, resizedBuffer);
+ const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));
+ return { filepath, bytes, width, height };
+ }
+
+ const outputFilePath = newPath.replace(extension, '.webp');
+ const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+ await fs.promises.writeFile(outputFilePath, data);
+ const bytes = Buffer.byteLength(data);
+ const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));
+ await fs.promises.unlink(inputFilePath);
+ return { filepath, bytes, width, height };
+}
+
+/**
+ * Encodes an image file to base64.
+ * @param {string} imagePath - The path to the image file.
+ * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
+ */
+function encodeImage(imagePath) {
+ return new Promise((resolve, reject) => {
+ fs.readFile(imagePath, (err, data) => {
+ if (err) {
+ reject(err);
+ } else {
+ resolve(data.toString('base64'));
+ }
+ });
+ });
+}
+
+/**
+ * Local: Updates the file and encodes the image to base64,
+ * for image payload handling: tuple order of [filepath, base64].
+ * @param {Object} req - The request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
+ */
+async function prepareImagesLocal(req, file) {
+ const { publicPath, imageOutput } = req.app.locals.paths;
+ const userPath = path.join(imageOutput, req.user.id);
+
+ if (!fs.existsSync(userPath)) {
+ fs.mkdirSync(userPath, { recursive: true });
+ }
+ const filepath = path.join(publicPath, file.filepath);
+
+ const promises = [];
+ promises.push(updateFile({ file_id: file.file_id }));
+ promises.push(encodeImage(filepath));
+ return await Promise.all(promises);
+}
+
+/**
+ * Uploads a user's avatar to Firebase Storage and returns the URL.
+ * If the 'manual' flag is set to 'true', it also updates the user's avatar URL in the database.
+ *
+ * @param {object} params - The parameters object.
+ * @param {Buffer} params.buffer - The Buffer containing the avatar image in WebP format.
+ * @param {object} params.User - The User document (mongoose); TODO: remove direct use of Model, `User`
+ * @param {string} params.manual - A string flag indicating whether the update is manual ('true' or 'false').
+ * @returns {Promise<string>} - A promise that resolves with the URL of the uploaded avatar.
+ * @throws {Error} - Throws an error if Firebase is not initialized or if there is an error in uploading.
+ */
+async function processLocalAvatar({ buffer, User, manual }) {
+ const userDir = path.resolve(
+ __dirname,
+ '..',
+ '..',
+ '..',
+ '..',
+ '..',
+ 'client',
+ 'public',
+ 'images',
+ User._id.toString(),
+ );
+ const fileName = `avatar-${new Date().getTime()}.png`;
+ const urlRoute = `/images/${User._id.toString()}/${fileName}`;
+ const avatarPath = path.join(userDir, fileName);
+
+ await fs.promises.mkdir(userDir, { recursive: true });
+ await fs.promises.writeFile(avatarPath, buffer);
+
+ const isManual = manual === 'true';
+ let url = `${urlRoute}?manual=${isManual}`;
+
+ if (isManual) {
+ User.avatar = url;
+ await User.save();
+ }
+
+ return url;
+}
+
+module.exports = { uploadLocalImage, encodeImage, prepareImagesLocal, processLocalAvatar };
diff --git a/api/server/services/Files/Local/index.js b/api/server/services/Files/Local/index.js
new file mode 100644
index 0000000000..cb44238bcc
--- /dev/null
+++ b/api/server/services/Files/Local/index.js
@@ -0,0 +1,7 @@
+const images = require('./images');
+const crud = require('./crud');
+
+module.exports = {
+ ...crud,
+ ...images,
+};
diff --git a/api/server/services/Files/images/avatar.js b/api/server/services/Files/images/avatar.js
new file mode 100644
index 0000000000..490fc86171
--- /dev/null
+++ b/api/server/services/Files/images/avatar.js
@@ -0,0 +1,78 @@
+const sharp = require('sharp');
+const fs = require('fs').promises;
+const fetch = require('node-fetch');
+const User = require('~/models/User');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
+const { logger } = require('~/config');
+
+async function convertToWebP(inputBuffer) {
+ return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
+}
+
+/**
+ * Uploads an avatar image for a user. This function can handle various types of input (URL, Buffer, or File object),
+ * processes the image to a square format, converts it to WebP format, and then uses a specified file strategy for
+ * further processing. It performs validation on the user ID and the input type. The function can throw errors for
+ * invalid input types, fetching issues, or other processing errors.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The unique identifier of the user for whom the avatar is being uploaded.
+ * @param {FileSources} params.fileStrategy - The file handling strategy to use, determining how the avatar is processed.
+ * @param {(string|Buffer|File)} params.input - The input representing the avatar image. Can be a URL (string),
+ * a Buffer, or a File object.
+ * @param {string} params.manual - A string flag indicating whether the upload process is manual.
+ *
+ * @returns {Promise<any>}
+ * A promise that resolves to the result of the `processAvatar` function, specific to the chosen file
+ * strategy. Throws an error if any step in the process fails.
+ *
+ * @throws {Error} Throws an error if the user ID is undefined, the input type is invalid, the image fetching fails,
+ * or any other error occurs during the processing.
+ */
+async function uploadAvatar({ userId, fileStrategy, input, manual }) {
+ try {
+ if (userId === undefined) {
+ throw new Error('User ID is undefined');
+ }
+ const _id = userId;
+ // TODO: remove direct use of Model, `User`
+ const oldUser = await User.findOne({ _id });
+
+ let imageBuffer;
+ if (typeof input === 'string') {
+ const response = await fetch(input);
+
+ if (!response.ok) {
+ throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);
+ }
+ imageBuffer = await response.buffer();
+ } else if (input instanceof Buffer) {
+ imageBuffer = input;
+ } else if (typeof input === 'object' && input instanceof File) {
+ const fileContent = await fs.readFile(input.path);
+ imageBuffer = Buffer.from(fileContent);
+ } else {
+ throw new Error('Invalid input type. Expected URL, Buffer, or File.');
+ }
+
+ const { width, height } = await sharp(imageBuffer).metadata();
+ const minSize = Math.min(width, height);
+ const squaredBuffer = await sharp(imageBuffer)
+ .extract({
+ left: Math.floor((width - minSize) / 2),
+ top: Math.floor((height - minSize) / 2),
+ width: minSize,
+ height: minSize,
+ })
+ .toBuffer();
+
+ const webPBuffer = await convertToWebP(squaredBuffer);
+ const { processAvatar } = getStrategyFunctions(fileStrategy);
+ return await processAvatar({ buffer: webPBuffer, User: oldUser, manual });
+ } catch (error) {
+ logger.error('Error uploading the avatar:', error);
+ throw error;
+ }
+}
+
+module.exports = uploadAvatar;
diff --git a/api/server/services/Files/images/avatar/firebaseStrategy.js b/api/server/services/Files/images/avatar/firebaseStrategy.js
deleted file mode 100644
index 9c000b43ec..0000000000
--- a/api/server/services/Files/images/avatar/firebaseStrategy.js
+++ /dev/null
@@ -1,29 +0,0 @@
-const { ref, uploadBytes, getDownloadURL } = require('firebase/storage');
-const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');
-const { logger } = require('~/config');
-
-async function firebaseStrategy(userId, webPBuffer, oldUser, manual) {
- try {
- const storage = getFirebaseStorage();
- if (!storage) {
- throw new Error('Firebase is not initialized.');
- }
- const avatarRef = ref(storage, `images/${userId.toString()}/avatar`);
-
- await uploadBytes(avatarRef, webPBuffer);
- const urlFirebase = await getDownloadURL(avatarRef);
- const isManual = manual === 'true';
-
- const url = `${urlFirebase}?manual=${isManual}`;
- if (isManual) {
- oldUser.avatar = url;
- await oldUser.save();
- }
- return url;
- } catch (error) {
- logger.error('Error uploading profile picture:', error);
- throw error;
- }
-}
-
-module.exports = firebaseStrategy;
diff --git a/api/server/services/Files/images/avatar/localStrategy.js b/api/server/services/Files/images/avatar/localStrategy.js
deleted file mode 100644
index 021beda7d1..0000000000
--- a/api/server/services/Files/images/avatar/localStrategy.js
+++ /dev/null
@@ -1,32 +0,0 @@
-const fs = require('fs').promises;
-const path = require('path');
-
-async function localStrategy(userId, webPBuffer, oldUser, manual) {
- const userDir = path.resolve(
- __dirname,
- '..',
- '..',
- '..',
- '..',
- '..',
- '..',
- 'client',
- 'public',
- 'images',
- userId,
- );
- let avatarPath = path.join(userDir, 'avatar.png');
- const urlRoute = `/images/${userId}/avatar.png`;
- await fs.mkdir(userDir, { recursive: true });
- await fs.writeFile(avatarPath, webPBuffer);
- const isManual = manual === 'true';
- let url = `${urlRoute}?manual=${isManual}&timestamp=${new Date().getTime()}`;
- if (isManual) {
- oldUser.avatar = url;
- await oldUser.save();
- }
-
- return url;
-}
-
-module.exports = localStrategy;
diff --git a/api/server/services/Files/images/avatar/uploadAvatar.js b/api/server/services/Files/images/avatar/uploadAvatar.js
deleted file mode 100644
index 0726df9a4d..0000000000
--- a/api/server/services/Files/images/avatar/uploadAvatar.js
+++ /dev/null
@@ -1,63 +0,0 @@
-const sharp = require('sharp');
-const fetch = require('node-fetch');
-const fs = require('fs').promises;
-const User = require('~/models/User');
-const { getFirebaseStorage } = require('~/server/services/Files/Firebase/initialize');
-const firebaseStrategy = require('./firebaseStrategy');
-const localStrategy = require('./localStrategy');
-const { logger } = require('~/config');
-
-async function convertToWebP(inputBuffer) {
- return sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
-}
-
-async function uploadAvatar(userId, input, manual) {
- try {
- if (userId === undefined) {
- throw new Error('User ID is undefined');
- }
- const _id = userId;
- // TODO: remove direct use of Model, `User`
- const oldUser = await User.findOne({ _id });
- let imageBuffer;
- if (typeof input === 'string') {
- const response = await fetch(input);
-
- if (!response.ok) {
- throw new Error(`Failed to fetch image from URL. Status: ${response.status}`);
- }
- imageBuffer = await response.buffer();
- } else if (input instanceof Buffer) {
- imageBuffer = input;
- } else if (typeof input === 'object' && input instanceof File) {
- const fileContent = await fs.readFile(input.path);
- imageBuffer = Buffer.from(fileContent);
- } else {
- throw new Error('Invalid input type. Expected URL, Buffer, or File.');
- }
- const { width, height } = await sharp(imageBuffer).metadata();
- const minSize = Math.min(width, height);
- const squaredBuffer = await sharp(imageBuffer)
- .extract({
- left: Math.floor((width - minSize) / 2),
- top: Math.floor((height - minSize) / 2),
- width: minSize,
- height: minSize,
- })
- .toBuffer();
- const webPBuffer = await convertToWebP(squaredBuffer);
- const storage = getFirebaseStorage();
- if (storage) {
- const url = await firebaseStrategy(userId, webPBuffer, oldUser, manual);
- return url;
- }
-
- const url = await localStrategy(userId, webPBuffer, oldUser, manual);
- return url;
- } catch (error) {
- logger.error('Error uploading the avatar:', error);
- throw error;
- }
-}
-
-module.exports = uploadAvatar;
diff --git a/api/server/services/Files/images/convert.js b/api/server/services/Files/images/convert.js
deleted file mode 100644
index 2de0fd2177..0000000000
--- a/api/server/services/Files/images/convert.js
+++ /dev/null
@@ -1,36 +0,0 @@
-const path = require('path');
-const sharp = require('sharp');
-const fs = require('fs');
-const { resizeImage } = require('./resize');
-
-async function convertToWebP(req, file, resolution = 'high') {
- const inputFilePath = file.path;
- const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
- const extension = path.extname(inputFilePath);
-
- const { imageOutput } = req.app.locals.config;
- const userPath = path.join(imageOutput, req.user.id);
-
- if (!fs.existsSync(userPath)) {
- fs.mkdirSync(userPath, { recursive: true });
- }
-
- const newPath = path.join(userPath, path.basename(inputFilePath));
-
- if (extension.toLowerCase() === '.webp') {
- const bytes = Buffer.byteLength(resizedBuffer);
- await fs.promises.writeFile(newPath, resizedBuffer);
- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(newPath));
- return { filepath, bytes, width, height };
- }
-
- const outputFilePath = newPath.replace(extension, '.webp');
- const data = await sharp(resizedBuffer).toFormat('webp').toBuffer();
- await fs.promises.writeFile(outputFilePath, data);
- const bytes = Buffer.byteLength(data);
- const filepath = path.posix.join('/', 'images', req.user.id, path.basename(outputFilePath));
- await fs.promises.unlink(inputFilePath);
- return { filepath, bytes, width, height };
-}
-
-module.exports = { convertToWebP };
diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js
index 30428ffabc..809ec0e840 100644
--- a/api/server/services/Files/images/encode.js
+++ b/api/server/services/Files/images/encode.js
@@ -1,45 +1,5 @@
-const fs = require('fs');
-const path = require('path');
-const { EModelEndpoint } = require('librechat-data-provider');
-const { updateFile } = require('~/models');
-
-/**
- * Encodes an image file to base64.
- * @param {string} imagePath - The path to the image file.
- * @returns {Promise<string>} A promise that resolves with the base64 encoded image data.
- */
-function encodeImage(imagePath) {
- return new Promise((resolve, reject) => {
- fs.readFile(imagePath, (err, data) => {
- if (err) {
- reject(err);
- } else {
- resolve(data.toString('base64'));
- }
- });
- });
-}
-
-/**
- * Updates the file and encodes the image.
- * @param {Object} req - The request object.
- * @param {Object} file - The file object.
- * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
- */
-async function updateAndEncode(req, file) {
- const { publicPath, imageOutput } = req.app.locals.config;
- const userPath = path.join(imageOutput, req.user.id);
-
- if (!fs.existsSync(userPath)) {
- fs.mkdirSync(userPath, { recursive: true });
- }
- const filepath = path.join(publicPath, file.filepath);
-
- const promises = [];
- promises.push(updateFile({ file_id: file.file_id }));
- promises.push(encodeImage(filepath));
- return await Promise.all(promises);
-}
+const { EModelEndpoint, FileSources } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('../strategies');
/**
* Encodes and formats the given files.
@@ -50,25 +10,42 @@ async function updateAndEncode(req, file) {
*/
async function encodeAndFormat(req, files, endpoint) {
const promises = [];
+ const encodingMethods = {};
+
for (let file of files) {
- promises.push(updateAndEncode(req, file));
+ const source = file.source ?? FileSources.local;
+
+ if (encodingMethods[source]) {
+ promises.push(encodingMethods[source](req, file));
+ continue;
+ }
+
+ const { prepareImagePayload } = getStrategyFunctions(source);
+ if (!prepareImagePayload) {
+ throw new Error(`Encoding function not implemented for ${source}`);
+ }
+
+ encodingMethods[source] = prepareImagePayload;
+ promises.push(prepareImagePayload(req, file));
}
- // TODO: make detail configurable, as of now resizing is done
- // to prefer "high" but "low" may be used if the image is small enough
- const detail = req.body.detail ?? 'auto';
- const encodedImages = await Promise.all(promises);
+ const detail = req.body.imageDetail ?? 'auto';
+
+ /** @type {Array<[MongoFile, string]>} */
+ const formattedImages = await Promise.all(promises);
const result = {
files: [],
image_urls: [],
};
- for (const [file, base64] of encodedImages) {
+ for (const [file, imageContent] of formattedImages) {
const imagePart = {
type: 'image_url',
image_url: {
- url: `data:image/webp;base64,${base64}`,
+ url: imageContent.startsWith('http')
+ ? imageContent
+ : `data:image/webp;base64,${imageContent}`,
detail,
},
};
@@ -81,17 +58,16 @@ async function encodeAndFormat(req, files, endpoint) {
result.files.push({
file_id: file.file_id,
- filepath: file.filepath,
- filename: file.filename,
- type: file.type,
- height: file.height,
- width: file.width,
+ // filepath: file.filepath,
+ // filename: file.filename,
+ // type: file.type,
+ // height: file.height,
+ // width: file.width,
});
}
return result;
}
module.exports = {
- encodeImage,
encodeAndFormat,
};
diff --git a/api/server/services/Files/images/index.js b/api/server/services/Files/images/index.js
index fa49eb9535..1438887e6d 100644
--- a/api/server/services/Files/images/index.js
+++ b/api/server/services/Files/images/index.js
@@ -1,15 +1,13 @@
-const convert = require('./convert');
+const avatar = require('./avatar');
const encode = require('./encode');
const parse = require('./parse');
const resize = require('./resize');
const validate = require('./validate');
-const uploadAvatar = require('./avatar/uploadAvatar');
module.exports = {
- ...convert,
...encode,
...parse,
...resize,
...validate,
- uploadAvatar,
+ avatar,
};
diff --git a/api/server/services/Files/images/parse.js b/api/server/services/Files/images/parse.js
index 5a1113c97e..1b0f7e4738 100644
--- a/api/server/services/Files/images/parse.js
+++ b/api/server/services/Files/images/parse.js
@@ -1,7 +1,7 @@
const URL = require('url').URL;
const path = require('path');
-const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg)$/i;
+const imageExtensionRegex = /\.(jpg|jpeg|png|gif|bmp|tiff|svg|webp)$/i;
/**
* Extracts the image basename from a given URL.
@@ -22,6 +22,24 @@ function getImageBasename(urlString) {
}
}
+/**
+ * Extracts the basename of a file from a given URL.
+ *
+ * @param {string} urlString - The URL string from which the file basename is to be extracted.
+ * @returns {string} The basename of the file from the URL.
+ * Returns an empty string if the URL parsing fails.
+ */
+function getFileBasename(urlString) {
+ try {
+ const url = new URL(urlString);
+ return path.basename(url.pathname);
+ } catch (error) {
+ // If URL parsing fails, return an empty string
+ return '';
+ }
+}
+
module.exports = {
getImageBasename,
+ getFileBasename,
};
diff --git a/api/server/services/Files/index.js b/api/server/services/Files/index.js
deleted file mode 100644
index 47d47690cb..0000000000
--- a/api/server/services/Files/index.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const localStrategy = require('./localStrategy');
-const process = require('./process');
-const save = require('./save');
-
-module.exports = {
- ...save,
- ...process,
- localStrategy,
-};
diff --git a/api/server/services/Files/localStrategy.js b/api/server/services/Files/localStrategy.js
deleted file mode 100644
index 0b711c7763..0000000000
--- a/api/server/services/Files/localStrategy.js
+++ /dev/null
@@ -1,36 +0,0 @@
-const { createFile } = require('~/models');
-const { convertToWebP } = require('./images/convert');
-
-/**
- * Applies the local strategy for image uploads.
- * Saves file metadata to the database with an expiry TTL.
- * Files must be deleted from the server filesystem manually.
- *
- * @param {Object} params - The parameters object.
- * @param {Express.Request} params.req - The Express request object.
- * @param {Express.Response} params.res - The Express response object.
- * @param {Express.Multer.File} params.file - The uploaded file.
- * @param {ImageMetadata} params.metadata - Additional metadata for the file.
- * @returns {Promise<void>}
- */
-const localStrategy = async ({ req, res, file, metadata }) => {
- const { file_id, temp_file_id } = metadata;
- const { filepath, bytes, width, height } = await convertToWebP(req, file);
- const result = await createFile(
- {
- user: req.user.id,
- file_id,
- temp_file_id,
- bytes,
- filepath,
- filename: file.originalname,
- type: 'image/webp',
- width,
- height,
- },
- true,
- );
- res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
-};
-
-module.exports = localStrategy;
diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js
index c6ab3ca139..4ee9510b4f 100644
--- a/api/server/services/Files/process.js
+++ b/api/server/services/Files/process.js
@@ -1,17 +1,6 @@
-const { updateFileUsage } = require('~/models');
-
-// const mapImageUrls = (files, detail) => {
-// return files
-// .filter((file) => file.type.includes('image'))
-// .map((file) => ({
-// type: 'image_url',
-// image_url: {
-// /* Temporarily set to path to encode later */
-// url: file.filepath,
-// detail,
-// },
-// }));
-// };
+const { updateFileUsage, createFile } = require('~/models');
+const { getStrategyFunctions } = require('./strategies');
+const { logger } = require('~/config');
const processFiles = async (files) => {
const promises = [];
@@ -24,6 +13,76 @@ const processFiles = async (files) => {
return await Promise.all(promises);
};
+/**
+ * Processes a file URL using a specified file handling strategy. This function accepts a strategy name,
+ * fetches the corresponding file processing functions (for saving and retrieving file URLs), and then
+ * executes these functions in sequence. It first saves the file using the provided URL and then retrieves
+ * the URL of the saved file. If any error occurs during this process, it logs the error and throws an
+ * exception with an appropriate message.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {FileSources} params.fileStrategy - The file handling strategy to use. Must be a value from the
+ * `FileSources` enum, which defines different file handling
+ * strategies (like saving to Firebase, local storage, etc.).
+ * @param {string} params.userId - The user's unique identifier. Used for creating user-specific paths or
+ * references in the file handling process.
+ * @param {string} params.URL - The URL of the file to be processed.
+ * @param {string} params.fileName - The name that will be used to save the file. This should include the
+ * file extension.
+ * @param {string} params.basePath - The base path or directory where the file will be saved or retrieved from.
+ *
+ * @returns {Promise<string>}
+ * A promise that resolves to the URL of the processed file. It throws an error if the file processing
+ * fails at any stage.
+ */
+const processFileURL = async ({ fileStrategy, userId, URL, fileName, basePath }) => {
+ const { saveURL, getFileURL } = getStrategyFunctions(fileStrategy);
+ try {
+ await saveURL({ userId, URL, fileName, basePath });
+ return await getFileURL({ fileName: `${userId}/${fileName}`, basePath });
+ } catch (error) {
+ logger.error(`Error while processing the image with ${fileStrategy}:`, error);
+ throw new Error(`Failed to process the image with ${fileStrategy}. ${error.message}`);
+ }
+};
+
+/**
+ * Applies the current strategy for image uploads.
+ * Saves file metadata to the database with an expiry TTL.
+ * Files must be deleted from the server filesystem manually.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {Express.Request} params.req - The Express request object.
+ * @param {Express.Response} params.res - The Express response object.
+ * @param {Express.Multer.File} params.file - The uploaded file.
+ * @param {ImageMetadata} params.metadata - Additional metadata for the file.
+ * @returns {Promise<void>}
+ */
+const processImageUpload = async ({ req, res, file, metadata }) => {
+ const source = req.app.locals.fileStrategy;
+ const { handleImageUpload } = getStrategyFunctions(source);
+ const { file_id, temp_file_id } = metadata;
+ const { filepath, bytes, width, height } = await handleImageUpload(req, file);
+ const result = await createFile(
+ {
+ user: req.user.id,
+ file_id,
+ temp_file_id,
+ bytes,
+ filepath,
+ filename: file.originalname,
+ source,
+ type: 'image/webp',
+ width,
+ height,
+ },
+ true,
+ );
+ res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
+};
+
module.exports = {
+ processImageUpload,
processFiles,
+ processFileURL,
};
diff --git a/api/server/services/Files/save.js b/api/server/services/Files/save.js
deleted file mode 100644
index 08f6a0d5cc..0000000000
--- a/api/server/services/Files/save.js
+++ /dev/null
@@ -1,48 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const { logger } = require('~/config');
-
-/**
- * Saves a file to a specified output path with a new filename.
- *
- * @param {Express.Multer.File} file - The file object to be saved. Should contain properties like 'originalname' and 'path'.
- * @param {string} outputPath - The path where the file should be saved.
- * @param {string} outputFilename - The new filename for the saved file (without extension).
- * @returns {Promise<string>} The full path of the saved file.
- * @throws Will throw an error if the file saving process fails.
- */
-async function saveFile(file, outputPath, outputFilename) {
- try {
- if (!fs.existsSync(outputPath)) {
- fs.mkdirSync(outputPath, { recursive: true });
- }
-
- const fileExtension = path.extname(file.originalname);
- const filenameWithExt = outputFilename + fileExtension;
- const outputFilePath = path.join(outputPath, filenameWithExt);
- fs.copyFileSync(file.path, outputFilePath);
- fs.unlinkSync(file.path);
-
- return outputFilePath;
- } catch (error) {
- logger.error('[saveFile] Error while saving the file:', error);
- throw error;
- }
-}
-
-/**
- * Saves an uploaded image file to a specified directory based on the user's ID and a filename.
- *
- * @param {Express.Request} req - The Express request object, containing the user's information and app configuration.
- * @param {Express.Multer.File} file - The uploaded file object.
- * @param {string} filename - The new filename to assign to the saved image (without extension).
- * @returns {Promise<void>}
- * @throws Will throw an error if the image saving process fails.
- */
-const saveLocalImage = async (req, file, filename) => {
- const imagePath = req.app.locals.config.imageOutput;
- const outputPath = path.join(imagePath, req.user.id ?? '');
- await saveFile(file, outputPath, filename);
-};
-
-module.exports = { saveFile, saveLocalImage };
diff --git a/api/server/services/Files/strategies.js b/api/server/services/Files/strategies.js
new file mode 100644
index 0000000000..4e20186043
--- /dev/null
+++ b/api/server/services/Files/strategies.js
@@ -0,0 +1,54 @@
+const { FileSources } = require('librechat-data-provider');
+const {
+ getFirebaseURL,
+ prepareImageURL,
+ saveURLToFirebase,
+ deleteFirebaseFile,
+ uploadImageToFirebase,
+ processFirebaseAvatar,
+} = require('./Firebase');
+const {
+ getLocalFileURL,
+ saveFileFromURL,
+ deleteLocalFile,
+ uploadLocalImage,
+ prepareImagesLocal,
+ processLocalAvatar,
+} = require('./Local');
+
+// Firebase Strategy Functions
+const firebaseStrategy = () => ({
+ // saveFile:
+ saveURL: saveURLToFirebase,
+ getFileURL: getFirebaseURL,
+ deleteFile: deleteFirebaseFile,
+ prepareImagePayload: prepareImageURL,
+ processAvatar: processFirebaseAvatar,
+ handleImageUpload: uploadImageToFirebase,
+});
+
+// Local Strategy Functions
+const localStrategy = () => ({
+ // saveFile: ,
+ saveURL: saveFileFromURL,
+ getFileURL: getLocalFileURL,
+ deleteFile: deleteLocalFile,
+ processAvatar: processLocalAvatar,
+ handleImageUpload: uploadLocalImage,
+ prepareImagePayload: prepareImagesLocal,
+});
+
+// Strategy Selector
+const getStrategyFunctions = (fileSource) => {
+ if (fileSource === FileSources.firebase) {
+ return firebaseStrategy();
+ } else if (fileSource === FileSources.local) {
+ return localStrategy();
+ } else {
+ throw new Error('Invalid file source');
+ }
+};
+
+module.exports = {
+ getStrategyFunctions,
+};
diff --git a/api/server/socialLogins.js b/api/server/socialLogins.js
index af61db73e9..4abe278b84 100644
--- a/api/server/socialLogins.js
+++ b/api/server/socialLogins.js
@@ -10,6 +10,10 @@ const {
} = require('../strategies');
const client = require('../cache/redis');
+/**
+ *
+ * @param {Express.Application} app
+ */
const configureSocialLogins = (app) => {
if (process.env.GOOGLE_CLIENT_ID && process.env.GOOGLE_CLIENT_SECRET) {
passport.use(googleLogin());
diff --git a/api/strategies/discordStrategy.js b/api/strategies/discordStrategy.js
index 994554200c..f7b042f940 100644
--- a/api/strategies/discordStrategy.js
+++ b/api/strategies/discordStrategy.js
@@ -1,12 +1,14 @@
const { Strategy: DiscordStrategy } = require('passport-discord');
+const { createNewUser, handleExistingUser } = require('./process');
const { logger } = require('~/config');
const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
const discordLogin = async (accessToken, refreshToken, profile, cb) => {
try {
const email = profile.email;
const discordId = profile.id;
+
+ // TODO: remove direct access of User model
const oldUser = await User.findOne({ email });
const ALLOW_SOCIAL_REGISTRATION =
process.env.ALLOW_SOCIAL_REGISTRATION?.toLowerCase() === 'true';
@@ -21,12 +23,20 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {
}
if (oldUser) {
- await handleExistingUser(oldUser, avatarUrl, useFirebase);
+ await handleExistingUser(oldUser, avatarUrl);
return cb(null, oldUser);
}
if (ALLOW_SOCIAL_REGISTRATION) {
- const newUser = await createNewUser(profile, discordId, email, avatarUrl, useFirebase);
+ const newUser = await createNewUser({
+ email,
+ avatarUrl,
+ provider: 'discord',
+ providerKey: 'discordId',
+ providerId: discordId,
+ username: profile.username,
+ name: profile.global_name,
+ });
return cb(null, newUser);
}
} catch (err) {
@@ -35,38 +45,6 @@ const discordLogin = async (accessToken, refreshToken, profile, cb) => {
}
};
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
- oldUser.avatar = avatarUrl;
- await oldUser.save();
- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
- const userId = oldUser._id;
- const newavatarUrl = await uploadAvatar(userId, avatarUrl);
- oldUser.avatar = newavatarUrl;
- await oldUser.save();
- }
-};
-
-const createNewUser = async (profile, discordId, email, avatarUrl, useFirebase) => {
- const newUser = await new User({
- provider: 'discord',
- discordId,
- username: profile.username,
- email,
- name: profile.global_name,
- avatar: avatarUrl,
- }).save();
-
- if (useFirebase) {
- const userId = newUser._id;
- const newavatarUrl = await uploadAvatar(userId, avatarUrl);
- newUser.avatar = newavatarUrl;
- await newUser.save();
- }
-
- return newUser;
-};
-
module.exports = () =>
new DiscordStrategy(
{
diff --git a/api/strategies/facebookStrategy.js b/api/strategies/facebookStrategy.js
index b8915b2cc4..dcf7beb797 100644
--- a/api/strategies/facebookStrategy.js
+++ b/api/strategies/facebookStrategy.js
@@ -1,7 +1,7 @@
const FacebookStrategy = require('passport-facebook').Strategy;
+const { createNewUser, handleExistingUser } = require('./process');
const { logger } = require('~/config');
const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
try {
@@ -13,12 +13,20 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
const avatarUrl = profile.photos[0]?.value;
if (oldUser) {
- await handleExistingUser(oldUser, avatarUrl, useFirebase);
+ await handleExistingUser(oldUser, avatarUrl);
return cb(null, oldUser);
}
if (ALLOW_SOCIAL_REGISTRATION) {
- const newUser = await createNewUser(profile, facebookId, email, avatarUrl, useFirebase);
+ const newUser = await createNewUser({
+ email,
+ avatarUrl,
+ provider: 'facebook',
+ providerKey: 'facebookId',
+ providerId: facebookId,
+ username: profile.displayName,
+ name: profile.name?.givenName + ' ' + profile.name?.familyName,
+ });
return cb(null, newUser);
}
} catch (err) {
@@ -27,38 +35,6 @@ const facebookLogin = async (accessToken, refreshToken, profile, cb) => {
}
};
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
- oldUser.avatar = avatarUrl;
- await oldUser.save();
- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
- const userId = oldUser._id;
- const newavatarUrl = await uploadAvatar(userId, avatarUrl);
- oldUser.avatar = newavatarUrl;
- await oldUser.save();
- }
-};
-
-const createNewUser = async (profile, facebookId, email, avatarUrl, useFirebase) => {
- const newUser = await new User({
- provider: 'facebook',
- facebookId,
- username: profile.displayName,
- email,
- name: profile.name?.givenName + ' ' + profile.name?.familyName,
- avatar: avatarUrl,
- }).save();
-
- if (useFirebase) {
- const userId = newUser._id;
- const newavatarUrl = await uploadAvatar(userId, avatarUrl);
- newUser.avatar = newavatarUrl;
- await newUser.save();
- }
-
- return newUser;
-};
-
module.exports = () =>
new FacebookStrategy(
{
diff --git a/api/strategies/githubStrategy.js b/api/strategies/githubStrategy.js
index c8480d50c1..065568f92f 100644
--- a/api/strategies/githubStrategy.js
+++ b/api/strategies/githubStrategy.js
@@ -1,7 +1,7 @@
const { Strategy: GitHubStrategy } = require('passport-github2');
+const { createNewUser, handleExistingUser } = require('./process');
const { logger } = require('~/config');
const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
const githubLogin = async (accessToken, refreshToken, profile, cb) => {
try {
@@ -13,12 +13,21 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {
const avatarUrl = profile.photos[0].value;
if (oldUser) {
- await handleExistingUser(oldUser, avatarUrl, useFirebase);
+ await handleExistingUser(oldUser, avatarUrl);
return cb(null, oldUser);
}
if (ALLOW_SOCIAL_REGISTRATION) {
- const newUser = await createNewUser(profile, githubId, email, avatarUrl, useFirebase);
+ const newUser = await createNewUser({
+ email,
+ avatarUrl,
+ provider: 'github',
+ providerKey: 'githubId',
+ providerId: githubId,
+ username: profile.username,
+ name: profile.displayName,
+ emailVerified: profile.emails[0].verified,
+ });
return cb(null, newUser);
}
} catch (err) {
@@ -27,39 +36,6 @@ const githubLogin = async (accessToken, refreshToken, profile, cb) => {
}
};
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
- if (!useFirebase && !oldUser.avatar.includes('?manual=true')) {
- oldUser.avatar = avatarUrl;
- await oldUser.save();
- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
- const userId = oldUser._id;
- const avatarURL = await uploadAvatar(userId, avatarUrl);
- oldUser.avatar = avatarURL;
- await oldUser.save();
- }
-};
-
-const createNewUser = async (profile, githubId, email, avatarUrl, useFirebase) => {
- const newUser = await new User({
- provider: 'github',
- githubId,
- username: profile.username,
- email,
- emailVerified: profile.emails[0].verified,
- name: profile.displayName,
- avatar: avatarUrl,
- }).save();
-
- if (useFirebase) {
- const userId = newUser._id;
- const avatarURL = await uploadAvatar(userId, avatarUrl);
- newUser.avatar = avatarURL;
- await newUser.save();
- }
-
- return newUser;
-};
-
module.exports = () =>
new GitHubStrategy(
{
diff --git a/api/strategies/googleStrategy.js b/api/strategies/googleStrategy.js
index d013cc8e8f..0eff48ee09 100644
--- a/api/strategies/googleStrategy.js
+++ b/api/strategies/googleStrategy.js
@@ -1,7 +1,7 @@
const { Strategy: GoogleStrategy } = require('passport-google-oauth20');
+const { createNewUser, handleExistingUser } = require('./process');
const { logger } = require('~/config');
const User = require('~/models/User');
-const { useFirebase, uploadAvatar } = require('~/server/services/Files/images');
const googleLogin = async (accessToken, refreshToken, profile, cb) => {
try {
@@ -13,12 +13,21 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {
const avatarUrl = profile.photos[0].value;
if (oldUser) {
- await handleExistingUser(oldUser, avatarUrl, useFirebase);
+ await handleExistingUser(oldUser, avatarUrl);
return cb(null, oldUser);
}
if (ALLOW_SOCIAL_REGISTRATION) {
- const newUser = await createNewUser(profile, googleId, email, avatarUrl, useFirebase);
+ const newUser = await createNewUser({
+ email,
+ avatarUrl,
+ provider: 'google',
+ providerKey: 'googleId',
+ providerId: googleId,
+ username: profile.name.givenName,
+ name: `${profile.name.givenName} ${profile.name.familyName}`,
+ emailVerified: profile.emails[0].verified,
+ });
return cb(null, newUser);
}
} catch (err) {
@@ -27,39 +36,6 @@ const googleLogin = async (accessToken, refreshToken, profile, cb) => {
}
};
-const handleExistingUser = async (oldUser, avatarUrl, useFirebase) => {
- if ((!useFirebase && !oldUser.avatar.includes('?manual=true')) || oldUser.avatar === null) {
- oldUser.avatar = avatarUrl;
- await oldUser.save();
- } else if (useFirebase && !oldUser.avatar.includes('?manual=true')) {
- const userId = oldUser._id;
- const avatarURL = await uploadAvatar(userId, avatarUrl);
- oldUser.avatar = avatarURL;
- await oldUser.save();
- }
-};
-
-const createNewUser = async (profile, googleId, email, avatarUrl, useFirebase) => {
- const newUser = await new User({
- provider: 'google',
- googleId,
- username: profile.name.givenName,
- email,
- emailVerified: profile.emails[0].verified,
- name: `${profile.name.givenName} ${profile.name.familyName}`,
- avatar: avatarUrl,
- }).save();
-
- if (useFirebase) {
- const userId = newUser._id;
- const avatarURL = await uploadAvatar(userId, avatarUrl);
- newUser.avatar = avatarURL;
- await newUser.save();
- }
-
- return newUser;
-};
-
module.exports = () =>
new GoogleStrategy(
{
diff --git a/api/strategies/process.js b/api/strategies/process.js
new file mode 100644
index 0000000000..f5a12a26a2
--- /dev/null
+++ b/api/strategies/process.js
@@ -0,0 +1,92 @@
+const { FileSources } = require('librechat-data-provider');
+const uploadAvatar = require('~/server/services/Files/images/avatar');
+const User = require('~/models/User');
+
+/**
+ * Updates the avatar URL of an existing user. If the user's avatar URL does not include the query parameter
+ * '?manual=true', it updates the user's avatar with the provided URL. For local file storage, it directly updates
+ * the avatar URL, while for other storage types, it processes the avatar URL using the specified file strategy.
+ *
+ * @param {User} oldUser - The existing user object that needs to be updated. Expected to have an 'avatar' property.
+ * @param {string} avatarUrl - The new avatar URL to be set for the user.
+ *
+ * @returns {Promise<void>}
+ * The function updates the user's avatar and saves the user object. It does not return any value.
+ *
+ * @throws {Error} Throws an error if there's an issue saving the updated user object.
+ */
+const handleExistingUser = async (oldUser, avatarUrl) => {
+ const fileStrategy = process.env.CDN_PROVIDER;
+ const isLocal = fileStrategy === FileSources.local;
+
+ if (isLocal && !oldUser.avatar.includes('?manual=true')) {
+ oldUser.avatar = avatarUrl;
+ await oldUser.save();
+ } else if (!isLocal && !oldUser.avatar.includes('?manual=true')) {
+ const userId = oldUser._id;
+ const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });
+ oldUser.avatar = newavatarUrl;
+ await oldUser.save();
+ }
+};
+
+/**
+ * Creates a new user with the provided user details. If the file strategy is not local, the avatar URL is
+ * processed using the specified file strategy. The new user is saved to the database with the processed or
+ * original avatar URL.
+ *
+ * @param {Object} params - The parameters object for user creation.
+ * @param {string} params.email - The email of the new user.
+ * @param {string} params.avatarUrl - The avatar URL of the new user.
+ * @param {string} params.provider - The provider of the user's account.
+ * @param {string} params.providerKey - The key to identify the provider in the user model.
+ * @param {string} params.providerId - The provider-specific ID of the user.
+ * @param {string} params.username - The username of the new user.
+ * @param {string} params.name - The name of the new user.
+ * @param {boolean} [params.emailVerified=false] - Optional. Indicates whether the user's email is verified. Defaults to false.
+ *
+ * @returns {Promise<User>}
+ * A promise that resolves to the newly created user object.
+ *
+ * @throws {Error} Throws an error if there's an issue creating or saving the new user object.
+ */
+const createNewUser = async ({
+ email,
+ avatarUrl,
+ provider,
+ providerKey,
+ providerId,
+ username,
+ name,
+ emailVerified,
+}) => {
+ const update = {
+ email,
+ avatar: avatarUrl,
+ provider,
+ [providerKey]: providerId,
+ username,
+ name,
+ emailVerified,
+ };
+
+ // TODO: remove direct access of User model
+ const newUser = await new User(update).save();
+
+ const fileStrategy = process.env.CDN_PROVIDER;
+ const isLocal = fileStrategy === FileSources.local;
+
+ if (!isLocal) {
+ const userId = newUser._id;
+ const newavatarUrl = await uploadAvatar({ userId, input: avatarUrl, fileStrategy });
+ newUser.avatar = newavatarUrl;
+ await newUser.save();
+ }
+
+ return newUser;
+};
+
+module.exports = {
+ handleExistingUser,
+ createNewUser,
+};
diff --git a/api/test/__mocks__/logger.js b/api/test/__mocks__/logger.js
index 455ada0de0..caeb004e39 100644
--- a/api/test/__mocks__/logger.js
+++ b/api/test/__mocks__/logger.js
@@ -1,3 +1,43 @@
+jest.mock('winston', () => {
+ const mockFormatFunction = jest.fn((fn) => fn);
+
+ mockFormatFunction.colorize = jest.fn();
+ mockFormatFunction.combine = jest.fn();
+ mockFormatFunction.label = jest.fn();
+ mockFormatFunction.timestamp = jest.fn();
+ mockFormatFunction.printf = jest.fn();
+ mockFormatFunction.errors = jest.fn();
+ mockFormatFunction.splat = jest.fn();
+ return {
+ format: mockFormatFunction,
+ createLogger: jest.fn().mockReturnValue({
+ info: jest.fn(),
+ warn: jest.fn(),
+ debug: jest.fn(),
+ error: jest.fn(),
+ }),
+ transports: {
+ Console: jest.fn(),
+ DailyRotateFile: jest.fn(),
+ },
+ addColors: jest.fn(),
+ };
+});
+
+jest.mock('winston-daily-rotate-file', () => {
+ return jest.fn().mockImplementation(() => {
+ return {
+ level: 'error',
+ filename: '../logs/error-%DATE%.log',
+ datePattern: 'YYYY-MM-DD',
+ zippedArchive: true,
+ maxSize: '20m',
+ maxFiles: '14d',
+ format: 'format',
+ };
+ });
+});
+
jest.mock('~/config', () => {
return {
logger: {
@@ -8,3 +48,11 @@ jest.mock('~/config', () => {
},
};
});
+
+jest.mock('~/config/parsers', () => {
+ return {
+ redactMessage: jest.fn(),
+ redactFormat: jest.fn(),
+ debugTraverse: jest.fn(),
+ };
+});
diff --git a/api/typedefs.js b/api/typedefs.js
index e40a097634..7bb956c9ae 100644
--- a/api/typedefs.js
+++ b/api/typedefs.js
@@ -26,6 +26,18 @@
* @memberof typedefs
*/
+/**
+ * @exports TMessage
+ * @typedef {import('librechat-data-provider').TMessage} TMessage
+ * @memberof typedefs
+ */
+
+/**
+ * @exports FileSources
+ * @typedef {import('librechat-data-provider').FileSources} FileSources
+ * @memberof typedefs
+ */
+
/**
* @exports ImageMetadata
* @typedef {Object} ImageMetadata
diff --git a/client/src/common/types.ts b/client/src/common/types.ts
index 17625731ff..1ca169a0c1 100644
--- a/client/src/common/types.ts
+++ b/client/src/common/types.ts
@@ -1,3 +1,4 @@
+import { FileSources } from 'librechat-data-provider';
import type {
TConversation,
TMessage,
@@ -230,6 +231,7 @@ export interface ExtendedFile {
size: number;
preview: string;
progress: number;
+ source?: FileSources;
}
export type ContextType = { navVisible: boolean; setNavVisible: (visible: boolean) => void };
diff --git a/client/src/components/Chat/ChatView.tsx b/client/src/components/Chat/ChatView.tsx
index d582f4b3e9..30a7edc187 100644
--- a/client/src/components/Chat/ChatView.tsx
+++ b/client/src/components/Chat/ChatView.tsx
@@ -6,11 +6,12 @@ import { useChatHelpers, useSSE } from '~/hooks';
// import GenerationButtons from './Input/GenerationButtons';
import MessagesView from './Messages/MessagesView';
// import OptionsBar from './Input/OptionsBar';
+import { useGetFiles } from '~/data-provider';
+import { buildTree, mapFiles } from '~/utils';
import { Spinner } from '~/components/svg';
import { ChatContext } from '~/Providers';
import Presentation from './Presentation';
import ChatForm from './Input/ChatForm';
-import { buildTree } from '~/utils';
import Landing from './Landing';
import Header from './Header';
import Footer from './Footer';
@@ -21,11 +22,16 @@ function ChatView({ index = 0 }: { index?: number }) {
const submissionAtIndex = useRecoilValue(store.submissionByIndex(0));
useSSE(submissionAtIndex);
+ const { data: fileMap } = useGetFiles({
+ select: mapFiles,
+ });
+
const { data: messagesTree = null, isLoading } = useGetMessagesByConvoId(conversationId ?? '', {
select: (data) => {
- const dataTree = buildTree(data, false);
+ const dataTree = buildTree({ messages: data, fileMap });
return dataTree?.length === 0 ? null : dataTree ?? null;
},
+ enabled: !!fileMap,
});
const chatHelpers = useChatHelpers(index, conversationId);
diff --git a/client/src/components/Chat/Input/Files/Images.tsx b/client/src/components/Chat/Input/Files/Images.tsx
index 421f1f69cd..903bd96c71 100644
--- a/client/src/components/Chat/Input/Files/Images.tsx
+++ b/client/src/components/Chat/Input/Files/Images.tsx
@@ -1,5 +1,6 @@
import debounce from 'lodash/debounce';
import { useState, useEffect, useCallback } from 'react';
+import { FileSources } from 'librechat-data-provider';
import type { BatchFile } from 'librechat-data-provider';
import { useDeleteFilesMutation } from '~/data-provider';
import { useSetFilesToDelete } from '~/hooks';
@@ -70,13 +71,20 @@ export default function Images({
}
const deleteFile = (_file: ExtendedFile) => {
- const { file_id, progress, temp_file_id = '', filepath = '' } = _file;
+ const {
+ file_id,
+ progress,
+ temp_file_id = '',
+ filepath = '',
+ source = FileSources.local,
+ } = _file;
if (progress < 1) {
return;
}
- const file = {
+ const file: BatchFile = {
file_id,
filepath,
+ source,
};
setFiles((currentFiles) => {
diff --git a/client/src/components/Chat/Messages/Content/MessageContent.tsx b/client/src/components/Chat/Messages/Content/MessageContent.tsx
index 875d4ebb5a..2415259aea 100644
--- a/client/src/components/Chat/Messages/Content/MessageContent.tsx
+++ b/client/src/components/Chat/Messages/Content/MessageContent.tsx
@@ -30,7 +30,7 @@ const ErrorMessage = ({ text }: TText) => {
// Display Message Component
const DisplayMessage = ({ text, isCreatedByUser, message, showCursor }: TDisplayProps) => {
const imageFiles = message?.files
- ? message.files.filter((file) => file.type.startsWith('image/'))
+ ? message.files.filter((file) => file.type && file.type.startsWith('image/'))
: null;
return (
<Container>
diff --git a/client/src/components/Chat/Presentation.tsx b/client/src/components/Chat/Presentation.tsx
index ea9d77f3b1..af264f1dc4 100644
--- a/client/src/components/Chat/Presentation.tsx
+++ b/client/src/components/Chat/Presentation.tsx
@@ -1,4 +1,5 @@
import { useEffect } from 'react';
+import { FileSources } from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
import { useDragHelpers, useSetFilesToDelete } from '~/hooks';
import DragDropOverlay from './Input/Files/DragDropOverlay';
@@ -25,6 +26,7 @@ export default function Presentation({ children }: { children: React.ReactNode }
.map((file) => ({
file_id: file.file_id,
filepath: file.filepath as string,
+ source: file.source as FileSources,
}));
if (files.length === 0) {
diff --git a/client/src/components/Endpoints/PopoverButtons.tsx b/client/src/components/Endpoints/PopoverButtons.tsx
index 217cce1a52..f763770fca 100644
--- a/client/src/components/Endpoints/PopoverButtons.tsx
+++ b/client/src/components/Endpoints/PopoverButtons.tsx
@@ -18,7 +18,7 @@ export default function PopoverButtons({
buttonClass,
iconClass = '',
}: {
- endpoint: EModelEndpoint;
+ endpoint: EModelEndpoint | string;
buttonClass?: string;
iconClass?: string;
}) {
@@ -32,7 +32,9 @@ export default function PopoverButtons({
const buttons: { [key: string]: TPopoverButton[] } = {
google: [
{
- label: (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) + localize('com_endpoint_examples'),
+ label:
+ (showExamples ? localize('com_endpoint_hide') : localize('com_endpoint_show')) +
+ localize('com_endpoint_examples'),
buttonClass: isCodeChat ? 'disabled' : '',
handler: triggerExamples,
icon: <MessagesSquared className={cn('mr-1 w-[14px]', iconClass)} />,
@@ -40,7 +42,10 @@ export default function PopoverButtons({
],
gptPlugins: [
{
- label: localize('com_endpoint_show_what_settings', showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent')),
+ label: localize(
+ 'com_endpoint_show_what_settings',
+ showAgentSettings ? localize('com_endpoint_completion') : localize('com_endpoint_agent'),
+ ),
buttonClass: '',
handler: () => setShowAgentSettings((prev) => !prev),
icon: <GPTIcon className={cn('mr-1 w-[14px]', iconClass)} size={24} />,
diff --git a/client/src/components/Endpoints/Settings/OpenAI.tsx b/client/src/components/Endpoints/Settings/OpenAI.tsx
index 77b224c8d3..652d46c586 100644
--- a/client/src/components/Endpoints/Settings/OpenAI.tsx
+++ b/client/src/components/Endpoints/Settings/OpenAI.tsx
@@ -1,11 +1,13 @@
import TextareaAutosize from 'react-textarea-autosize';
+import { ImageDetail, imageDetailNumeric, imageDetailValue } from 'librechat-data-provider';
import {
- SelectDropDown,
Input,
Label,
+ Switch,
Slider,
- InputNumber,
HoverCard,
+ InputNumber,
+ SelectDropDown,
HoverCardTrigger,
} from '~/components/ui';
import { cn, defaultTextProps, optionText, removeFocusOutlines } from '~/utils/';
@@ -20,6 +22,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
return null;
}
const {
+ endpoint,
+ endpointType,
model,
chatGptLabel,
promptPrefix,
@@ -27,6 +31,8 @@ export default function Settings({ conversation, setOption, models, readonly }:
top_p: topP,
frequency_penalty: freqP,
presence_penalty: presP,
+ resendImages,
+ imageDetail,
} = conversation;
const setModel = setOption('model');
const setChatGptLabel = setOption('chatGptLabel');
@@ -35,6 +41,10 @@ export default function Settings({ conversation, setOption, models, readonly }:
const setTopP = setOption('top_p');
const setFreqP = setOption('frequency_penalty');
const setPresP = setOption('presence_penalty');
+ const setResendImages = setOption('resendImages');
+ const setImageDetail = setOption('imageDetail');
+
+ const optionEndpoint = endpointType ?? endpoint;
return (
<div className="grid grid-cols-5 gap-6">
@@ -126,7 +136,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
- <OptionHover endpoint={conversation?.endpoint ?? ''} type="temp" side={ESide.Left} />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="temp" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
<HoverCardTrigger className="grid w-full items-center gap-2">
@@ -164,7 +174,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
- <OptionHover endpoint={conversation?.endpoint ?? ''} type="topp" side={ESide.Left} />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="topp" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
@@ -203,7 +213,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
- <OptionHover endpoint={conversation?.endpoint ?? ''} type="freq" side={ESide.Left} />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="freq" side={ESide.Left} />
</HoverCard>
<HoverCard openDelay={300}>
@@ -242,8 +252,66 @@ export default function Settings({ conversation, setOption, models, readonly }:
className="flex h-4 w-full"
/>
</HoverCardTrigger>
- <OptionHover endpoint={conversation?.endpoint ?? ''} type="pres" side={ESide.Left} />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="pres" side={ESide.Left} />
</HoverCard>
+ <div className="w-full">
+ <div className="mb-2 flex w-full justify-between gap-2">
+ <label
+ htmlFor="resend-images"
+ className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
+ >
+ <small>{localize('com_endpoint_plug_resend_images')}</small>
+ </label>
+ <label
+ htmlFor="image-detail-value"
+ className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
+ >
+ <small>Image Detail</small>
+ </label>
+ <Input
+ id="image-detail-value"
+ disabled={true}
+ value={imageDetail ?? ImageDetail.auto}
+ className={cn(
+ defaultTextProps,
+ optionText,
+ 'flex rounded-md bg-transparent py-2 text-xs focus:outline-none focus:ring-2 focus:ring-slate-400 focus:ring-offset-2 dark:border-slate-700',
+ 'pointer-events-none max-h-5 w-12 border-0 group-hover/temp:border-gray-200',
+ )}
+ />
+ </div>
+ <div className="flex w-full justify-between gap-2">
+ <HoverCard openDelay={500}>
+ <HoverCardTrigger>
+ <Switch
+ id="resend-images"
+ checked={resendImages ?? false}
+ onCheckedChange={(checked: boolean) => setResendImages(checked)}
+ disabled={readonly}
+ className="flex"
+ />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="resend" side={ESide.Bottom} />
+ </HoverCardTrigger>
+ </HoverCard>
+ <HoverCard openDelay={500}>
+ <HoverCardTrigger className="flex w-[52%] md:w-[125px]">
+ <Slider
+ id="image-detail-slider"
+ disabled={readonly}
+ value={[
+ imageDetailNumeric[imageDetail ?? ''] ?? imageDetailNumeric[ImageDetail.auto],
+ ]}
+ onValueChange={(value) => setImageDetail(imageDetailValue[value[0]])}
+ doubleClickHandler={() => setImageDetail(ImageDetail.auto)}
+ max={2}
+ min={0}
+ step={1}
+ />
+ <OptionHover endpoint={optionEndpoint ?? ''} type="detail" side={ESide.Bottom} />
+ </HoverCardTrigger>
+ </HoverCard>
+ </div>
+ </div>
</div>
</div>
);
diff --git a/client/src/components/Endpoints/Settings/OptionHover.tsx b/client/src/components/Endpoints/Settings/OptionHover.tsx
index 2648a06184..f6af64c149 100644
--- a/client/src/components/Endpoints/Settings/OptionHover.tsx
+++ b/client/src/components/Endpoints/Settings/OptionHover.tsx
@@ -15,6 +15,8 @@ const openAI = {
topp: 'com_endpoint_openai_topp',
freq: 'com_endpoint_openai_freq',
pres: 'com_endpoint_openai_pres',
+ resend: 'com_endpoint_openai_resend',
+ detail: 'com_endpoint_openai_detail',
};
const types = {
@@ -47,7 +49,7 @@ function OptionHover({ endpoint, type, side }: TOptionHoverProps) {
}
return (
<HoverCardPortal>
- <HoverCardContent side={side} className="w-80 ">
+ <HoverCardContent side={side} className="z-[80] w-80">
<div className="space-y-2">
<p className="text-sm text-gray-600 dark:text-gray-300">{localize(text)}</p>
</div>
diff --git a/client/src/components/Nav/ExportConversation/ExportModal.jsx b/client/src/components/Nav/ExportConversation/ExportModal.jsx
index e4f292a78a..d2cfb49c1e 100644
--- a/client/src/components/Nav/ExportConversation/ExportModal.jsx
+++ b/client/src/components/Nav/ExportConversation/ExportModal.jsx
@@ -24,7 +24,7 @@ export default function ExportModal({ open, onOpenChange, conversation }) {
const { data: messagesTree = null } = useGetMessagesByConvoId(conversation.conversationId ?? '', {
select: (data) => {
- const dataTree = buildTree(data, false);
+ const dataTree = buildTree({ messages: data });
return dataTree?.length === 0 ? null : dataTree ?? null;
},
});
diff --git a/client/src/data-provider/mutations.ts b/client/src/data-provider/mutations.ts
index 236c66a947..e607818cc7 100644
--- a/client/src/data-provider/mutations.ts
+++ b/client/src/data-provider/mutations.ts
@@ -1,7 +1,7 @@
import { useMutation, useQueryClient } from '@tanstack/react-query';
import type { UseMutationResult } from '@tanstack/react-query';
import type {
- FileUploadResponse,
+ TFileUpload,
UploadMutationOptions,
FileUploadBody,
DeleteFilesResponse,
@@ -23,7 +23,7 @@ import store from '~/store';
export const useUploadImageMutation = (
options?: UploadMutationOptions,
): UseMutationResult<
- FileUploadResponse, // response data
+ TFileUpload, // response data
unknown, // error
FileUploadBody, // request
unknown // context
diff --git a/client/src/data-provider/queries.ts b/client/src/data-provider/queries.ts
index 4144b64812..2ae6aaf81b 100644
--- a/client/src/data-provider/queries.ts
+++ b/client/src/data-provider/queries.ts
@@ -1,6 +1,18 @@
-import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';
import { QueryKeys, dataService } from 'librechat-data-provider';
-import type { TPreset } from 'librechat-data-provider';
+import { UseQueryOptions, useQuery, QueryObserverResult } from '@tanstack/react-query';
+import type { TPreset, TFile } from 'librechat-data-provider';
+
+export const useGetFiles = <TData = TFile[] | boolean>(
+ config?: UseQueryOptions<TFile[], unknown, TData>,
+): QueryObserverResult<TData, unknown> => {
+ return useQuery<TFile[], unknown, TData>([QueryKeys.files], () => dataService.getFiles(), {
+ refetchOnWindowFocus: false,
+ refetchOnReconnect: false,
+ refetchOnMount: false,
+ ...config,
+ });
+};
+
export const useGetPresetsQuery = (
config?: UseQueryOptions<TPreset[]>,
): QueryObserverResult<TPreset[], unknown> => {
diff --git a/client/src/hooks/useFileHandling.ts b/client/src/hooks/useFileHandling.ts
index 21126a98f8..bd6874336e 100644
--- a/client/src/hooks/useFileHandling.ts
+++ b/client/src/hooks/useFileHandling.ts
@@ -1,6 +1,9 @@
import { v4 } from 'uuid';
import debounce from 'lodash/debounce';
+import { QueryKeys } from 'librechat-data-provider';
+import { useQueryClient } from '@tanstack/react-query';
import { useState, useEffect, useCallback } from 'react';
+import type { TFile } from 'librechat-data-provider';
import type { ExtendedFile } from '~/common';
import { useToastContext } from '~/Providers/ToastContext';
import { useChatContext } from '~/Providers/ChatContext';
@@ -16,6 +19,7 @@ const totalSizeLimit = maxSize * 1024 * 1024; // 25 MB
const supportedTypes = ['image/jpeg', 'image/jpg', 'image/png', 'image/webp'];
const useFileHandling = () => {
+ const queryClient = useQueryClient();
const { showToast } = useToastContext();
const [errors, setErrors] = useState<string[]>([]);
const setError = (error: string) => setErrors((prevErrors) => [...prevErrors, error]);
@@ -116,6 +120,9 @@ const useFileHandling = () => {
filepath: data.filepath,
});
+ const _files = queryClient.getQueryData<TFile[]>([QueryKeys.files]) ?? [];
+ queryClient.setQueryData([QueryKeys.files], [..._files, data]);
+
setTimeout(() => {
updateFileById(data.temp_file_id, {
progress: 1,
@@ -126,6 +133,7 @@ const useFileHandling = () => {
height: data.height,
width: data.width,
filename: data.filename,
+ source: data.source,
});
}, 300);
},
diff --git a/client/src/hooks/useNewConvo.ts b/client/src/hooks/useNewConvo.ts
index 897636e587..d6c3bba8c3 100644
--- a/client/src/hooks/useNewConvo.ts
+++ b/client/src/hooks/useNewConvo.ts
@@ -1,4 +1,5 @@
import { useCallback } from 'react';
+import { FileSources } from 'librechat-data-provider';
import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
import {
useSetRecoilState,
@@ -122,10 +123,11 @@ const useNewConvo = (index = 0) => {
if (conversation.conversationId === 'new' && !modelsData) {
const filesToDelete = Array.from(files.values())
- .filter((file) => file.filepath)
+ .filter((file) => file.filepath && file.source)
.map((file) => ({
file_id: file.file_id,
filepath: file.filepath as string,
+ source: file.source as FileSources, // Ensure that the source is of type FileSources
}));
setFiles(new Map());
diff --git a/client/src/localization/languages/Eng.tsx b/client/src/localization/languages/Eng.tsx
index f084cf50f8..52749713e3 100644
--- a/client/src/localization/languages/Eng.tsx
+++ b/client/src/localization/languages/Eng.tsx
@@ -162,6 +162,10 @@ export default {
'Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model\'s likelihood to repeat the same line verbatim.',
com_endpoint_openai_pres:
'Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model\'s likelihood to talk about new topics.',
+ com_endpoint_openai_resend:
+ 'Resend all previously attached images. Note: this can significantly increase token cost and you may experience errors with many image attachments.',
+ com_endpoint_openai_detail:
+ 'The resolution for Vision requests. "Low" is cheaper and faster, "High" is more detailed and expensive, and "Auto" will automatically choose between the two based on the image resolution.',
com_endpoint_openai_custom_name_placeholder: 'Set a custom name for ChatGPT',
com_endpoint_openai_prompt_prefix_placeholder:
'Set custom instructions to include in System Message. Default: none',
@@ -177,6 +181,7 @@ export default {
com_endpoint_frequency_penalty: 'Frequency Penalty',
com_endpoint_presence_penalty: 'Presence Penalty',
com_endpoint_plug_use_functions: 'Use Functions',
+ com_endpoint_plug_resend_images: 'Resend Images',
com_endpoint_plug_skip_completion: 'Skip Completion',
com_endpoint_disabled_with_tools: 'disabled with tools',
com_endpoint_disabled_with_tools_placeholder: 'Disabled with Tools Selected',
diff --git a/client/src/routes/ChatRoute.tsx b/client/src/routes/ChatRoute.tsx
index 8cca53233b..aaf0e5ab8c 100644
--- a/client/src/routes/ChatRoute.tsx
+++ b/client/src/routes/ChatRoute.tsx
@@ -6,7 +6,7 @@ import {
useGetModelsQuery,
useGetEndpointsQuery,
} from 'librechat-data-provider/react-query';
-import { TPreset } from 'librechat-data-provider';
+import type { TPreset } from 'librechat-data-provider';
import { useNewConvo, useConfigOverride } from '~/hooks';
import ChatView from '~/components/Chat/ChatView';
import useAuthRedirect from './useAuthRedirect';
@@ -15,6 +15,7 @@ import store from '~/store';
export default function ChatRoute() {
const index = 0;
+
useConfigOverride();
const { conversationId } = useParams();
const { conversation } = store.useCreateConversationAtom(index);
diff --git a/client/src/routes/Root.tsx b/client/src/routes/Root.tsx
index 7da118ba97..28224de3fc 100644
--- a/client/src/routes/Root.tsx
+++ b/client/src/routes/Root.tsx
@@ -1,11 +1,12 @@
/* eslint-disable react-hooks/exhaustive-deps */
import { useEffect, useState } from 'react';
-import { useRecoilValue, useSetRecoilState } from 'recoil';
import { Outlet, useLocation } from 'react-router-dom';
+import { useRecoilValue, useSetRecoilState } from 'recoil';
import { useGetModelsQuery, useGetSearchEnabledQuery } from 'librechat-data-provider/react-query';
import type { ContextType } from '~/common';
import { useAuthContext, useServerStream, useConversation } from '~/hooks';
import { Nav, MobileNav } from '~/components/Nav';
+import { useGetFiles } from '~/data-provider';
import store from '~/store';
export default function Root() {
@@ -24,6 +25,7 @@ export default function Root() {
const setIsSearchEnabled = useSetRecoilState(store.isSearchEnabled);
const setModelsConfig = useSetRecoilState(store.modelsConfig);
+ useGetFiles({ enabled: isAuthenticated });
const searchEnabledQuery = useGetSearchEnabledQuery({ enabled: isAuthenticated });
const modelsQuery = useGetModelsQuery({ enabled: isAuthenticated && modelsQueryEnabled });
diff --git a/client/src/store/conversation.ts b/client/src/store/conversation.ts
index a575ac385d..ee12d71496 100644
--- a/client/src/store/conversation.ts
+++ b/client/src/store/conversation.ts
@@ -18,7 +18,7 @@ const messages = atom<TMessagesAtom>({
const messagesTree = selector({
key: 'messagesTree',
get: ({ get }) => {
- return buildTree(get(messages), false);
+ return buildTree({ messages: get(messages) });
},
});
diff --git a/client/src/store/search.ts b/client/src/store/search.ts
index 4f4099b473..b687058e87 100644
--- a/client/src/store/search.ts
+++ b/client/src/store/search.ts
@@ -20,7 +20,7 @@ const searchResultMessages = atom<TMessage[] | null>({
const searchResultMessagesTree = selector({
key: 'searchResultMessagesTree',
get: ({ get }) => {
- return buildTree(get(searchResultMessages), true);
+ return buildTree({ messages: get(searchResultMessages), groupAll: true });
},
});
diff --git a/client/src/utils/buildTree.ts b/client/src/utils/buildTree.ts
index 23b74a11a0..637801b548 100644
--- a/client/src/utils/buildTree.ts
+++ b/client/src/utils/buildTree.ts
@@ -1,11 +1,19 @@
-import { TMessage } from 'librechat-data-provider';
+import { TFile, TMessage } from 'librechat-data-provider';
const even =
'w-full border-b border-black/10 dark:border-gray-900/50 text-gray-800 bg-white dark:text-gray-100 group dark:bg-gray-800 hover:bg-gray-100/25 hover:text-gray-700 dark:hover:bg-gray-900 dark:hover:text-gray-200';
const odd =
'w-full border-b border-black/10 bg-gray-50 dark:border-gray-900/50 text-gray-800 dark:text-gray-100 group bg-gray-100 dark:bg-gray-1000 hover:bg-gray-100/40 hover:text-gray-700 dark:hover:bg-[#3b3d49] dark:hover:text-gray-200';
-export default function buildTree(messages: TMessage[] | null, groupAll = false) {
+export default function buildTree({
+ messages,
+ fileMap,
+ groupAll = false,
+}: {
+ messages: TMessage[] | null;
+ fileMap?: Record<string, TFile>;
+ groupAll?: boolean;
+}) {
if (messages === null) {
return null;
}
@@ -21,6 +29,12 @@ export default function buildTree(messages: TMessage[] | null, groupAll = false)
messages.forEach((message) => {
messageMap[message.messageId] = { ...message, children: [] };
+ if (message.files && fileMap) {
+ messageMap[message.messageId].files = message.files.map(
+ (file) => fileMap[file.file_id] ?? file,
+ );
+ }
+
const parentMessage = messageMap[message.parentMessageId ?? ''];
if (parentMessage) {
parentMessage.children.push(messageMap[message.messageId]);
diff --git a/client/src/utils/files.ts b/client/src/utils/files.ts
new file mode 100644
index 0000000000..4615029fff
--- /dev/null
+++ b/client/src/utils/files.ts
@@ -0,0 +1,12 @@
+import type { TFile } from 'librechat-data-provider';
+
+/** Maps Files by `file_id` for quick lookup */
+export function mapFiles(files: TFile[]) {
+ const fileMap = {} as Record<string, TFile>;
+
+ for (const file of files) {
+ fileMap[file.file_id] = file;
+ }
+
+ return fileMap;
+}
diff --git a/client/src/utils/index.ts b/client/src/utils/index.ts
index bf7ab065c4..12a32faf63 100644
--- a/client/src/utils/index.ts
+++ b/client/src/utils/index.ts
@@ -1,4 +1,5 @@
export * from './json';
+export * from './files';
export * from './presets';
export * from './languages';
export * from './endpoints';
diff --git a/client/src/utils/presets.ts b/client/src/utils/presets.ts
index ee86087d87..45a7cd21fd 100644
--- a/client/src/utils/presets.ts
+++ b/client/src/utils/presets.ts
@@ -12,6 +12,8 @@ export const getPresetIcon = (preset: TPreset, Icon) => {
});
};
+type TEndpoints = Array<string | EModelEndpoint>;
+
export const getPresetTitle = (preset: TPreset) => {
const {
endpoint,
@@ -26,12 +28,16 @@ export const getPresetTitle = (preset: TPreset) => {
let modelInfo = model || '';
let label = '';
- if (
- endpoint &&
- [EModelEndpoint.azureOpenAI, EModelEndpoint.openAI, EModelEndpoint.custom].includes(endpoint)
- ) {
+ const usesChatGPTLabel: TEndpoints = [
+ EModelEndpoint.azureOpenAI,
+ EModelEndpoint.openAI,
+ EModelEndpoint.custom,
+ ];
+ const usesModelLabel: TEndpoints = [EModelEndpoint.google, EModelEndpoint.anthropic];
+
+ if (endpoint && usesChatGPTLabel.includes(endpoint)) {
label = chatGptLabel || '';
- } else if (endpoint && [EModelEndpoint.google, EModelEndpoint.anthropic].includes(endpoint)) {
+ } else if (endpoint && usesModelLabel.includes(endpoint)) {
label = modelLabel || '';
} else if (endpoint === EModelEndpoint.bingAI) {
modelInfo = jailbreak ? 'Sydney' : modelInfo;
diff --git a/docs/features/firebase.md b/docs/features/firebase.md
index fc94b3bce9..e48ab9a2e7 100644
--- a/docs/features/firebase.md
+++ b/docs/features/firebase.md
@@ -100,4 +100,20 @@ FIREBASE_APP_ID=1:your_app_id #appId
- Publish your updated rules
-![image](https://github.com/danny-avila/LibreChat/assets/32828263/5e6a17c3-5aba-419a-a18f-be910b1f25d5)
\ No newline at end of file
+![image](https://github.com/danny-avila/LibreChat/assets/32828263/5e6a17c3-5aba-419a-a18f-be910b1f25d5)
+
+### Configure `fileStrategy` in `librechat.yaml`
+
+Finally, to enable the app use Firebase, you must set the following in your `librechat.yaml` config file.
+
+```yaml
+ version: 1.0.1
+ cache: true
+ fileStrategy: "firebase" # This is the field and value you need to add
+ endpoints:
+ custom:
+ - name: "Mistral"
+ # Rest of file omitted
+```
+
+For more information about the `librechat.yaml` config file, [see the guide here](../install/configuration/custom_config.md).
\ No newline at end of file
diff --git a/docs/features/index.md b/docs/features/index.md
index 9bfda2be25..e025a83d2b 100644
--- a/docs/features/index.md
+++ b/docs/features/index.md
@@ -8,6 +8,7 @@ weight: 2
---
+* 🤖[Custom Endpoints](../install/configuration/custom_config.md)
* 🔌[Plugins](./plugins/index.md)
* 🔌 [Introduction](./plugins/introduction.md)
* 🛠️ [Make Your Own](./plugins/make_your_own.md)
diff --git a/docs/install/configuration/custom_config.md b/docs/install/configuration/custom_config.md
index 0e62330977..b9802fe051 100644
--- a/docs/install/configuration/custom_config.md
+++ b/docs/install/configuration/custom_config.md
@@ -69,20 +69,26 @@ docker-compose up # no need to rebuild
**Note:** Fields not specifically mentioned as required are optional.
-### 1. Version
+### Version
- **Key**: `version`
- **Type**: String
- **Description**: Specifies the version of the configuration file.
-- **Example**: `version: 1.0.0`
+- **Example**: `version: 1.0.1`
- **Required**
-### 2. Cache Settings
+### Cache Settings
- **Key**: `cache`
- **Type**: Boolean
- **Description**: Toggles caching on or off. Set to `true` to enable caching.
- **Example**: `cache: true`
-### 3. Endpoints
+### File Strategy
+- **Key**: `fileStrategy`
+- **Type**: String = "local" | ["firebase"](../../features/firebase.md)
+- **Description**: Determines where to save user uploaded/generated files. Defaults to `"local"` if omitted.
+- **Example**: `fileStrategy: "firebase"`
+
+### Endpoints
- **Key**: `endpoints`
- **Type**: Object
- **Description**: Defines custom API endpoints for the application.
@@ -290,7 +296,7 @@ Custom endpoints share logic with the OpenAI endpoint, and thus have default par
## Example Config
```yaml
-version: 1.0.0
+version: 1.0.1
cache: true
endpoints:
custom:
diff --git a/librechat.example.yaml b/librechat.example.yaml
index b2a9bbaf93..047cfbcb30 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -1,5 +1,5 @@
# Configuration version (required)
-version: 1.0.0
+version: 1.0.1
# Cache settings: Set to true to enable caching
cache: true
diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json
index c0b0a05209..9148305ce1 100644
--- a/packages/data-provider/package.json
+++ b/packages/data-provider/package.json
@@ -1,6 +1,6 @@
{
"name": "librechat-data-provider",
- "version": "0.3.5",
+ "version": "0.3.6",
"description": "data services for librechat apps",
"main": "dist/index.js",
"module": "dist/index.es.js",
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 7b31508cc5..a7cc732d16 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1,5 +1,8 @@
import { z } from 'zod';
import { EModelEndpoint, eModelEndpointSchema } from './schemas';
+import { FileSources } from './types/files';
+
+export const fileSourceSchema = z.nativeEnum(FileSources);
export const endpointSchema = z.object({
name: z.string().refine((value) => !eModelEndpointSchema.safeParse(value).success, {
@@ -25,6 +28,7 @@ export const endpointSchema = z.object({
export const configSchema = z.object({
version: z.string(),
cache: z.boolean(),
+ fileStrategy: fileSourceSchema.optional(),
endpoints: z
.object({
custom: z.array(endpointSchema.partial()),
@@ -184,3 +188,27 @@ export enum AuthKeys {
*/
GOOGLE_API_KEY = 'GOOGLE_API_KEY',
}
+
+/**
+ * Enum for Image Detail Cost.
+ *
+ * **Low Res Fixed Cost:** `85`
+ *
+ * **High Res Calculation:**
+ *
+ * Number of `512px` Tiles * `170` + `85` (Additional Cost)
+ */
+export enum ImageDetailCost {
+ /**
+ * Low resolution is a fixed value.
+ */
+ LOW = 85,
+ /**
+ * High resolution Cost Per Tile
+ */
+ HIGH = 170,
+ /**
+ * Additional Cost added to High Resolution Total Cost
+ */
+ ADDITIONAL = 85,
+}
diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts
index 74ad766575..f6b4221404 100644
--- a/packages/data-provider/src/data-service.ts
+++ b/packages/data-provider/src/data-service.ts
@@ -193,7 +193,11 @@ export const listAssistants = (
/* Files */
-export const uploadImage = (data: FormData): Promise<f.FileUploadResponse> => {
+export const getFiles = (): Promise<f.TFile[]> => {
+ return request.get(endpoints.files());
+};
+
+export const uploadImage = (data: FormData): Promise<f.TFileUpload> => {
return request.postMultiPart(endpoints.images(), data);
};
diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.ts
index ec150d9587..b0368f5b96 100644
--- a/packages/data-provider/src/keys.ts
+++ b/packages/data-provider/src/keys.ts
@@ -16,6 +16,7 @@ export enum QueryKeys {
assistants = 'assistants',
assistant = 'assistant',
endpointsConfigOverride = 'endpointsConfigOverride',
+ files = 'files',
}
export enum MutationKeys {
diff --git a/packages/data-provider/src/schemas.ts b/packages/data-provider/src/schemas.ts
index caa951fa6a..25cf3825c7 100644
--- a/packages/data-provider/src/schemas.ts
+++ b/packages/data-provider/src/schemas.ts
@@ -52,6 +52,26 @@ export const eModelEndpointSchema = z.nativeEnum(EModelEndpoint);
export const extendedModelEndpointSchema = z.union([eModelEndpointSchema, z.string()]);
+export enum ImageDetail {
+ low = 'low',
+ auto = 'auto',
+ high = 'high',
+}
+
+export const imageDetailNumeric = {
+ [ImageDetail.low]: 0,
+ [ImageDetail.auto]: 1,
+ [ImageDetail.high]: 2,
+};
+
+export const imageDetailValue = {
+ 0: ImageDetail.low,
+ 1: ImageDetail.auto,
+ 2: ImageDetail.high,
+};
+
+export const eImageDetailSchema = z.nativeEnum(ImageDetail);
+
export const tPluginAuthConfigSchema = z.object({
authField: z.string(),
label: z.string(),
@@ -140,8 +160,8 @@ export type TMessage = z.input<typeof tMessageSchema> & {
plugin?: TResPlugin | null;
plugins?: TResPlugin[];
files?: {
- type: string;
file_id: string;
+ type?: string;
filename?: string;
preview?: string;
filepath?: string;
@@ -184,6 +204,9 @@ export const tConversationSchema = z.object({
toneStyle: z.string().nullable().optional(),
maxOutputTokens: z.number().optional(),
agentOptions: tAgentOptionsSchema.nullable().optional(),
+ /* vision */
+ resendImages: z.boolean().optional(),
+ imageDetail: eImageDetailSchema.optional(),
/* assistant */
assistant_id: z.string().optional(),
thread_id: z.string().optional(),
@@ -234,6 +257,8 @@ export const openAISchema = tConversationSchema
top_p: true,
presence_penalty: true,
frequency_penalty: true,
+ resendImages: true,
+ imageDetail: true,
})
.transform((obj) => ({
...obj,
@@ -244,6 +269,8 @@ export const openAISchema = tConversationSchema
top_p: obj.top_p ?? 1,
presence_penalty: obj.presence_penalty ?? 0,
frequency_penalty: obj.frequency_penalty ?? 0,
+ resendImages: obj.resendImages ?? false,
+ imageDetail: obj.imageDetail ?? ImageDetail.auto,
}))
.catch(() => ({
model: 'gpt-3.5-turbo',
@@ -253,6 +280,8 @@ export const openAISchema = tConversationSchema
top_p: 1,
presence_penalty: 0,
frequency_penalty: 0,
+ resendImages: false,
+ imageDetail: ImageDetail.auto,
}));
export const googleSchema = tConversationSchema
@@ -455,6 +484,8 @@ export const compactOpenAISchema = tConversationSchema
top_p: true,
presence_penalty: true,
frequency_penalty: true,
+ resendImages: true,
+ imageDetail: true,
})
.transform((obj: Partial<TConversation>) => {
const newObj: Partial<TConversation> = { ...obj };
@@ -473,6 +504,12 @@ export const compactOpenAISchema = tConversationSchema
if (newObj.frequency_penalty === 0) {
delete newObj.frequency_penalty;
}
+ if (newObj.resendImages !== true) {
+ delete newObj.resendImages;
+ }
+ if (newObj.imageDetail === ImageDetail.auto) {
+ delete newObj.imageDetail;
+ }
return removeNullishValues(newObj);
})
diff --git a/packages/data-provider/src/types/files.ts b/packages/data-provider/src/types/files.ts
index 95f329f652..dadf5cc640 100644
--- a/packages/data-provider/src/types/files.ts
+++ b/packages/data-provider/src/types/files.ts
@@ -1,13 +1,25 @@
-export type FileUploadResponse = {
+export enum FileSources {
+ local = 'local',
+ firebase = 'firebase',
+ openai = 'openai',
+ s3 = 's3',
+}
+
+export type TFile = {
message: string;
file_id: string;
- temp_file_id: string;
filepath: string;
filename: string;
type: string;
size: number;
- height: number;
- width: number;
+ temp_file_id?: string;
+ source?: FileSources;
+ height?: number;
+ width?: number;
+};
+
+export type TFileUpload = TFile & {
+ temp_file_id: string;
};
export type AvatarUploadResponse = {
@@ -20,7 +32,7 @@ export type FileUploadBody = {
};
export type UploadMutationOptions = {
- onSuccess?: (data: FileUploadResponse, variables: FileUploadBody, context?: unknown) => void;
+ onSuccess?: (data: TFileUpload, variables: FileUploadBody, context?: unknown) => void;
onMutate?: (variables: FileUploadBody) => void | Promise<unknown>;
onError?: (error: unknown, variables: FileUploadBody, context?: unknown) => void;
};
@@ -39,6 +51,7 @@ export type DeleteFilesResponse = {
export type BatchFile = {
file_id: string;
filepath: string;
+ source: FileSources;
};
export type DeleteFilesBody = {