Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🎨 fix: Optimize StableDiffusion API Tool and Fix for Assistants Usage (#2253)
Danny Avilabb8a40dd983d
diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.jsindex 75c23e923f..3155992ca9 100644--- a/api/app/clients/tools/structured/DALLE3.js+++ b/api/app/clients/tools/structured/DALLE3.js@@ -12,14 +12,15 @@ const { logger } = require('~/config');class DALLE3 extends Tool {constructor(fields = {}) {super();- /* Used to initialize the Tool without necessary variables. */+ /** @type {boolean} Used to initialize the Tool without necessary variables. */this.override = fields.override ?? false;- /* Necessary for output to contain all image metadata. */+ /** @type {boolean} Necessary for output to contain all image metadata. */this.returnMetadata = fields.returnMetadata ?? false;this.userId = fields.userId;this.fileStrategy = fields.fileStrategy;if (fields.processFileURL) {+ /** @type {processFileURL} Necessary for output to contain all image metadata. */this.processFileURL = fields.processFileURL.bind(this);}@@ -165,13 +166,7 @@ Error Message: ${error.message}`;});if (this.returnMetadata) {- this.result = {- file_id: result.file_id,- filename: result.filename,- filepath: result.filepath,- height: result.height,- width: result.width,- };+ this.result = result;} else {this.result = this.wrapInMarkdown(result.filepath);}diff --git a/api/app/clients/tools/structured/StableDiffusion.js b/api/app/clients/tools/structured/StableDiffusion.jsindex dc479037b5..6e15db72b8 100644--- a/api/app/clients/tools/structured/StableDiffusion.js+++ b/api/app/clients/tools/structured/StableDiffusion.js@@ -4,14 +4,27 @@ const { z } = require('zod');const path = require('path');const axios = require('axios');const sharp = require('sharp');+const { v4: uuidv4 } = require('uuid');const { StructuredTool } = require('langchain/tools');+const { FileContext } = require('librechat-data-provider');+const paths = require('~/config/paths');const { logger } = require('~/config');class StableDiffusionAPI extends StructuredTool {constructor(fields) {super();- /* Used to initialize the Tool without necessary variables. */+ /** @type {string} User ID */+ this.userId = fields.userId;+ /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */+ this.req = fields.req;+ /** @type {boolean} Used to initialize the Tool without necessary variables. */this.override = fields.override ?? false;+ /** @type {boolean} Necessary for output to contain all image metadata. */+ this.returnMetadata = fields.returnMetadata ?? false;+ if (fields.uploadImageBuffer) {+ /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */+ this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);+ }this.name = 'stable-diffusion';this.url = fields.SD_WEBUI_URL || this.getServerURL();@@ -47,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {getMarkdownImageUrl(imageName) {const imageUrl = path- .join(this.relativeImageUrl, imageName)+ .join(this.relativePath, this.userId, imageName).replace(/\\/g, '/').replace('public/', '');return ``;@@ -73,46 +86,67 @@ class StableDiffusionAPI extends StructuredTool {width: 1024,height: 1024,};- const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);- const image = response.data.images[0];- const pngPayload = { image: `data:image/png;base64,${image}` };- const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);- const info = response2.data.info;+ const generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);+ const image = generationResponse.data.images[0];++ /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */+ let info = {};+ try {+ info = JSON.parse(generationResponse.data.info);+ } catch (error) {+ logger.error('[StableDiffusion] Error while getting image metadata:', error);+ }- // Generate unique name- const imageName = `${Date.now()}.png`;- this.outputPath = path.resolve(- __dirname,- '..',- '..',- '..',- '..',- '..',- 'client',- 'public',- 'images',- );- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');- this.relativeImageUrl = path.relative(appRoot, this.outputPath);+ const file_id = uuidv4();+ const imageName = `${file_id}.png`;+ const { imageOutput: imageOutputPath, clientPath } = paths;+ const filepath = path.join(imageOutputPath, this.userId, imageName);+ this.relativePath = path.relative(clientPath, imageOutputPath);- // Check if directory exists, if not create it- if (!fs.existsSync(this.outputPath)) {- fs.mkdirSync(this.outputPath, { recursive: true });+ if (!fs.existsSync(imageOutputPath)) {+ fs.mkdirSync(imageOutputPath, { recursive: true });}try {const buffer = Buffer.from(image.split(',', 1)[0], 'base64');+ if (this.returnMetadata && this.uploadImageBuffer && this.req) {+ const file = await this.uploadImageBuffer({+ req: this.req,+ context: FileContext.image_generation,+ resize: false,+ metadata: {+ buffer,+ height: info.height,+ width: info.width,+ bytes: Buffer.byteLength(buffer),+ filename: imageName,+ type: 'image/png',+ file_id,+ },+ });++ const generationInfo = info.infotexts[0].split('\n').pop();+ return {+ ...file,+ prompt,+ metadata: {+ negative_prompt,+ seed: info.seed,+ info: generationInfo,+ },+ };+ }+await sharp(buffer).withMetadata({iptcpng: {- parameters: info,+ parameters: info.infotexts[0],},})- .toFile(this.outputPath + '/' + imageName);+ .toFile(filepath);this.result = this.getMarkdownImageUrl(imageName);} catch (error) {logger.error('[StableDiffusion] Error while saving the image:', error);- // this.result = theImageUrl;}return this.result;diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.jsindex 6c9e43f03d..7ed1865871 100644--- a/api/app/clients/tools/util/handleTools.js+++ b/api/app/clients/tools/util/handleTools.js@@ -237,9 +237,11 @@ const loadTools = async ({}const imageGenOptions = {+ req: options.req,fileStrategy: options.fileStrategy,processFileURL: options.processFileURL,returnMetadata: options.returnMetadata,+ uploadImageBuffer: options.uploadImageBuffer,};const toolOptions = {diff --git a/api/config/paths.js b/api/config/paths.jsindex 92921218e8..49c3f1fc79 100644--- a/api/config/paths.js+++ b/api/config/paths.js@@ -1,7 +1,9 @@const path = require('path');module.exports = {+ root: path.resolve(__dirname, '..', '..'),uploads: path.resolve(__dirname, '..', '..', 'uploads'),+ clientPath: path.resolve(__dirname, '..', '..', 'client'),dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),diff --git a/api/server/services/Files/images/resize.js b/api/server/services/Files/images/resize.jsindex 89d3ef692d..ac05ba9946 100644--- a/api/server/services/Files/images/resize.js+++ b/api/server/services/Files/images/resize.js@@ -62,14 +62,14 @@ async function resizeImageBuffer(inputBuffer, resolution, endpoint) {}/**- * Resizes an image buffer to webp format as well as reduces 150 px width.+ * Resizes an image buffer to webp format as well as reduces by specified or default 150 px width.** @param {Buffer} inputBuffer - The buffer of the image to be resized.* @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size and dimensions.* @throws Will throw an error if the resolution parameter is invalid.*/-async function resizeAndConvert(inputBuffer) {- const resizedBuffer = await sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();+async function resizeAndConvert(inputBuffer, width = 150) {+ const resizedBuffer = await sharp(inputBuffer).resize({ width }).toFormat('webp').toBuffer();const resizedMetadata = await sharp(resizedBuffer).metadata();return {buffer: resizedBuffer,diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.jsindex 983cc06db7..66a5e45499 100644--- a/api/server/services/Files/process.js+++ b/api/server/services/Files/process.js@@ -223,26 +223,32 @@ const processImageFile = async ({ req, res, file, metadata }) => {* @param {Object} params - The parameters object.* @param {Express.Request} params.req - The Express request object.* @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.)+ * @param {boolean} [params.resize=true] - Whether to resize and convert the image to WebP. Default is `true`.+ * @param {{ buffer: Buffer, width: number, height: number, bytes: number, filename: string, type: string, file_id: string }} [params.metadata] - Required metadata for the file if resize is false.* @returns {Promise<{ filepath: string, filename: string, source: string, type: 'image/webp'}>}*/-const uploadImageBuffer = async ({ req, context }) => {+const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) => {const source = req.app.locals.fileStrategy;const { saveBuffer } = getStrategyFunctions(source);- const { buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer);- const file_id = v4();- const fileName = `img-${file_id}.webp`;+ let { buffer, width, height, bytes, filename, file_id, type } = metadata;+ if (resize) {+ file_id = v4();+ type = 'image/webp';+ ({ buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer));+ filename = path.basename(req.file.originalname, path.extname(req.file.originalname)) + '.webp';+ }- const filepath = await saveBuffer({ userId: req.user.id, fileName, buffer });+ const filepath = await saveBuffer({ userId: req.user.id, fileName: filename, buffer });return await createFile({user: req.user.id,file_id,bytes,filepath,- filename: req.file.originalname,+ filename,context,source,- type: 'image/webp',+ type,width,height,},diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.jsindex 923e9a7ab0..5a94480383 100644--- a/api/server/services/ToolService.js+++ b/api/server/services/ToolService.js@@ -12,8 +12,8 @@ const {openapiToFunction,validateAndParseOpenAPISpec,} = require('librechat-data-provider');+const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');const { loadActionSets, createActionTool, domainParser } = require('./ActionService');-const { processFileURL } = require('~/server/services/Files/process');const { recordUsage } = require('~/server/services/Threads');const { loadTools } = require('~/app/clients/tools/util');const { redactMessage } = require('~/config/parsers');@@ -147,7 +147,7 @@ const processVisionRequest = async (client, currentAction) => {/*** Processes return required actions from run.- * @param {OpenAIClient} client - OpenAI or StreamRunManager Client.+ * @param {OpenAIClient | StreamRunManager} client - OpenAI (legacy) or StreamRunManager Client.* @param {RequiredAction[]} requiredActions - The required actions to submit outputs for.* @returns {Promise<ToolOutputs>} The outputs of the tools.*/@@ -164,6 +164,8 @@ async function processRequiredActions(client, requiredActions) {functions: true,options: {processFileURL,+ req: client.req,+ uploadImageBuffer,openAIApiKey: client.apiKey,fileStrategy: client.req.app.locals.fileStrategy,returnMetadata: true,diff --git a/api/typedefs.js b/api/typedefs.jsindex 24870b63ba..c8ac7a5c88 100644--- a/api/typedefs.js+++ b/api/typedefs.js@@ -386,6 +386,18 @@* @memberof typedefs*/+/**+ * @exports uploadImageBuffer+ * @typedef {import('~/server/services/Files/process').uploadImageBuffer} uploadImageBuffer+ * @memberof typedefs+ */++/**+ * @exports processFileURL+ * @typedef {import('~/server/services/Files/process').processFileURL} processFileURL+ * @memberof typedefs+ */+/*** @exports AssistantCreateParams* @typedef {import('librechat-data-provider').AssistantCreateParams} AssistantCreateParamsdiff --git a/docs/install/installation/docker_compose_install.md b/docs/install/installation/docker_compose_install.mdindex 5b72f1cc62..9a14b48379 100644--- a/docs/install/installation/docker_compose_install.md+++ b/docs/install/installation/docker_compose_install.md@@ -51,7 +51,12 @@ Once you have completed all the setup, you can start the LibreChat application bThat's it! If you need more detailed information on configuring your compose file, see my notes below.## Updating LibreChat-The following commands will fetch the latest code of LibreChat and build a new docker image.++As of v0.7.0+, Docker installations transitioned from building images locally to using prebuilt images [hosted on Github Container registry](https://github.com/danny-avila?tab=packages&repo_name=LibreChat).++You can still build the image locally, as shown in the commented commands below. More info on building the image locally in the [Docker Compose Override Section](../configuration/docker_override.md).++The following commands will fetch the latest LibreChat project changes, including any necessary changes to the docker compose files, as well as the latest prebuilt images.```bash# Stop the running container(s)@@ -63,13 +68,16 @@ git pull# Pull the latest LibreChat image (default setup)docker compose pull+# If building the LibreChat image Locally, build without cache (legacy setup)+# docker compose build --no-cache+# Start LibreChatdocker compose up```-If you're having issues running the above commands, you can try a comprehensive approach:+If you're having issues running the above commands, you can try a comprehensive approach instead:-Prefix commands with `sudo` according to your environment permissions.+Note: you may need to prefix commands with `sudo` according to your environment permissions.```bash# Stop the container (if running)