Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🎨 fix: Optimize StableDiffusion API Tool and Fix for Assistants Usage (#2253)

Danny Avilabb8a40dd983d
diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js
index 75c23e923f..3155992ca9 100644
--- a/api/app/clients/tools/structured/DALLE3.js
+++ b/api/app/clients/tools/structured/DALLE3.js
@@ -12,14 +12,15 @@ const { logger } = require('~/config');
class DALLE3 extends Tool {
constructor(fields = {}) {
super();
- /* Used to initialize the Tool without necessary variables. */
+ /** @type {boolean} Used to initialize the Tool without necessary variables. */
this.override = fields.override ?? false;
- /* Necessary for output to contain all image metadata. */
+ /** @type {boolean} Necessary for output to contain all image metadata. */
this.returnMetadata = fields.returnMetadata ?? false;
this.userId = fields.userId;
this.fileStrategy = fields.fileStrategy;
if (fields.processFileURL) {
+ /** @type {processFileURL} Necessary for output to contain all image metadata. */
this.processFileURL = fields.processFileURL.bind(this);
}
@@ -165,13 +166,7 @@ Error Message: ${error.message}`;
});
if (this.returnMetadata) {
- this.result = {
- file_id: result.file_id,
- filename: result.filename,
- filepath: result.filepath,
- height: result.height,
- width: result.width,
- };
+ this.result = result;
} else {
this.result = this.wrapInMarkdown(result.filepath);
}
diff --git a/api/app/clients/tools/structured/StableDiffusion.js b/api/app/clients/tools/structured/StableDiffusion.js
index dc479037b5..6e15db72b8 100644
--- a/api/app/clients/tools/structured/StableDiffusion.js
+++ b/api/app/clients/tools/structured/StableDiffusion.js
@@ -4,14 +4,27 @@ const { z } = require('zod');
const path = require('path');
const axios = require('axios');
const sharp = require('sharp');
+const { v4: uuidv4 } = require('uuid');
const { StructuredTool } = require('langchain/tools');
+const { FileContext } = require('librechat-data-provider');
+const paths = require('~/config/paths');
const { logger } = require('~/config');
class StableDiffusionAPI extends StructuredTool {
constructor(fields) {
super();
- /* Used to initialize the Tool without necessary variables. */
+ /** @type {string} User ID */
+ this.userId = fields.userId;
+ /** @type {Express.Request | undefined} Express Request object, only provided by ToolService */
+ this.req = fields.req;
+ /** @type {boolean} Used to initialize the Tool without necessary variables. */
this.override = fields.override ?? false;
+ /** @type {boolean} Necessary for output to contain all image metadata. */
+ this.returnMetadata = fields.returnMetadata ?? false;
+ if (fields.uploadImageBuffer) {
+ /** @type {uploadImageBuffer} Necessary for output to contain all image metadata. */
+ this.uploadImageBuffer = fields.uploadImageBuffer.bind(this);
+ }
this.name = 'stable-diffusion';
this.url = fields.SD_WEBUI_URL || this.getServerURL();
@@ -47,7 +60,7 @@ class StableDiffusionAPI extends StructuredTool {
getMarkdownImageUrl(imageName) {
const imageUrl = path
- .join(this.relativeImageUrl, imageName)
+ .join(this.relativePath, this.userId, imageName)
.replace(/\\/g, '/')
.replace('public/', '');
return `![generated image](/${imageUrl})`;
@@ -73,46 +86,67 @@ class StableDiffusionAPI extends StructuredTool {
width: 1024,
height: 1024,
};
- const response = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
- const image = response.data.images[0];
- const pngPayload = { image: `data:image/png;base64,${image}` };
- const response2 = await axios.post(`${url}/sdapi/v1/png-info`, pngPayload);
- const info = response2.data.info;
+ const generationResponse = await axios.post(`${url}/sdapi/v1/txt2img`, payload);
+ const image = generationResponse.data.images[0];
+
+ /** @type {{ height: number, width: number, seed: number, infotexts: string[] }} */
+ let info = {};
+ try {
+ info = JSON.parse(generationResponse.data.info);
+ } catch (error) {
+ logger.error('[StableDiffusion] Error while getting image metadata:', error);
+ }
- // Generate unique name
- const imageName = `${Date.now()}.png`;
- this.outputPath = path.resolve(
- __dirname,
- '..',
- '..',
- '..',
- '..',
- '..',
- 'client',
- 'public',
- 'images',
- );
- const appRoot = path.resolve(__dirname, '..', '..', '..', '..', '..', 'client');
- this.relativeImageUrl = path.relative(appRoot, this.outputPath);
+ const file_id = uuidv4();
+ const imageName = `${file_id}.png`;
+ const { imageOutput: imageOutputPath, clientPath } = paths;
+ const filepath = path.join(imageOutputPath, this.userId, imageName);
+ this.relativePath = path.relative(clientPath, imageOutputPath);
- // Check if directory exists, if not create it
- if (!fs.existsSync(this.outputPath)) {
- fs.mkdirSync(this.outputPath, { recursive: true });
+ if (!fs.existsSync(imageOutputPath)) {
+ fs.mkdirSync(imageOutputPath, { recursive: true });
}
try {
const buffer = Buffer.from(image.split(',', 1)[0], 'base64');
+ if (this.returnMetadata && this.uploadImageBuffer && this.req) {
+ const file = await this.uploadImageBuffer({
+ req: this.req,
+ context: FileContext.image_generation,
+ resize: false,
+ metadata: {
+ buffer,
+ height: info.height,
+ width: info.width,
+ bytes: Buffer.byteLength(buffer),
+ filename: imageName,
+ type: 'image/png',
+ file_id,
+ },
+ });
+
+ const generationInfo = info.infotexts[0].split('\n').pop();
+ return {
+ ...file,
+ prompt,
+ metadata: {
+ negative_prompt,
+ seed: info.seed,
+ info: generationInfo,
+ },
+ };
+ }
+
await sharp(buffer)
.withMetadata({
iptcpng: {
- parameters: info,
+ parameters: info.infotexts[0],
},
})
- .toFile(this.outputPath + '/' + imageName);
+ .toFile(filepath);
this.result = this.getMarkdownImageUrl(imageName);
} catch (error) {
logger.error('[StableDiffusion] Error while saving the image:', error);
- // this.result = theImageUrl;
}
return this.result;
diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js
index 6c9e43f03d..7ed1865871 100644
--- a/api/app/clients/tools/util/handleTools.js
+++ b/api/app/clients/tools/util/handleTools.js
@@ -237,9 +237,11 @@ const loadTools = async ({
}
const imageGenOptions = {
+ req: options.req,
fileStrategy: options.fileStrategy,
processFileURL: options.processFileURL,
returnMetadata: options.returnMetadata,
+ uploadImageBuffer: options.uploadImageBuffer,
};
const toolOptions = {
diff --git a/api/config/paths.js b/api/config/paths.js
index 92921218e8..49c3f1fc79 100644
--- a/api/config/paths.js
+++ b/api/config/paths.js
@@ -1,7 +1,9 @@
const path = require('path');
module.exports = {
+ root: path.resolve(__dirname, '..', '..'),
uploads: path.resolve(__dirname, '..', '..', 'uploads'),
+ clientPath: path.resolve(__dirname, '..', '..', 'client'),
dist: path.resolve(__dirname, '..', '..', 'client', 'dist'),
publicPath: path.resolve(__dirname, '..', '..', 'client', 'public'),
imageOutput: path.resolve(__dirname, '..', '..', 'client', 'public', 'images'),
diff --git a/api/server/services/Files/images/resize.js b/api/server/services/Files/images/resize.js
index 89d3ef692d..ac05ba9946 100644
--- a/api/server/services/Files/images/resize.js
+++ b/api/server/services/Files/images/resize.js
@@ -62,14 +62,14 @@ async function resizeImageBuffer(inputBuffer, resolution, endpoint) {
}
/**
- * Resizes an image buffer to webp format as well as reduces 150 px width.
+ * Resizes an image buffer to webp format as well as reduces by specified or default 150 px width.
*
* @param {Buffer} inputBuffer - The buffer of the image to be resized.
* @returns {Promise<{ buffer: Buffer, width: number, height: number, bytes: number }>} An object containing the resized image buffer, its size and dimensions.
* @throws Will throw an error if the resolution parameter is invalid.
*/
-async function resizeAndConvert(inputBuffer) {
- const resizedBuffer = await sharp(inputBuffer).resize({ width: 150 }).toFormat('webp').toBuffer();
+async function resizeAndConvert(inputBuffer, width = 150) {
+ const resizedBuffer = await sharp(inputBuffer).resize({ width }).toFormat('webp').toBuffer();
const resizedMetadata = await sharp(resizedBuffer).metadata();
return {
buffer: resizedBuffer,
diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js
index 983cc06db7..66a5e45499 100644
--- a/api/server/services/Files/process.js
+++ b/api/server/services/Files/process.js
@@ -223,26 +223,32 @@ const processImageFile = async ({ req, res, file, metadata }) => {
* @param {Object} params - The parameters object.
* @param {Express.Request} params.req - The Express request object.
* @param {FileContext} params.context - The context of the file (e.g., 'avatar', 'image_generation', etc.)
+ * @param {boolean} [params.resize=true] - Whether to resize and convert the image to WebP. Default is `true`.
+ * @param {{ buffer: Buffer, width: number, height: number, bytes: number, filename: string, type: string, file_id: string }} [params.metadata] - Required metadata for the file if resize is false.
* @returns {Promise<{ filepath: string, filename: string, source: string, type: 'image/webp'}>}
*/
-const uploadImageBuffer = async ({ req, context }) => {
+const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) => {
const source = req.app.locals.fileStrategy;
const { saveBuffer } = getStrategyFunctions(source);
- const { buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer);
- const file_id = v4();
- const fileName = `img-${file_id}.webp`;
+ let { buffer, width, height, bytes, filename, file_id, type } = metadata;
+ if (resize) {
+ file_id = v4();
+ type = 'image/webp';
+ ({ buffer, width, height, bytes } = await resizeAndConvert(req.file.buffer));
+ filename = path.basename(req.file.originalname, path.extname(req.file.originalname)) + '.webp';
+ }
- const filepath = await saveBuffer({ userId: req.user.id, fileName, buffer });
+ const filepath = await saveBuffer({ userId: req.user.id, fileName: filename, buffer });
return await createFile(
{
user: req.user.id,
file_id,
bytes,
filepath,
- filename: req.file.originalname,
+ filename,
context,
source,
- type: 'image/webp',
+ type,
width,
height,
},
diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js
index 923e9a7ab0..5a94480383 100644
--- a/api/server/services/ToolService.js
+++ b/api/server/services/ToolService.js
@@ -12,8 +12,8 @@ const {
openapiToFunction,
validateAndParseOpenAPISpec,
} = require('librechat-data-provider');
+const { processFileURL, uploadImageBuffer } = require('~/server/services/Files/process');
const { loadActionSets, createActionTool, domainParser } = require('./ActionService');
-const { processFileURL } = require('~/server/services/Files/process');
const { recordUsage } = require('~/server/services/Threads');
const { loadTools } = require('~/app/clients/tools/util');
const { redactMessage } = require('~/config/parsers');
@@ -147,7 +147,7 @@ const processVisionRequest = async (client, currentAction) => {
/**
* Processes return required actions from run.
- * @param {OpenAIClient} client - OpenAI or StreamRunManager Client.
+ * @param {OpenAIClient | StreamRunManager} client - OpenAI (legacy) or StreamRunManager Client.
* @param {RequiredAction[]} requiredActions - The required actions to submit outputs for.
* @returns {Promise<ToolOutputs>} The outputs of the tools.
*/
@@ -164,6 +164,8 @@ async function processRequiredActions(client, requiredActions) {
functions: true,
options: {
processFileURL,
+ req: client.req,
+ uploadImageBuffer,
openAIApiKey: client.apiKey,
fileStrategy: client.req.app.locals.fileStrategy,
returnMetadata: true,
diff --git a/api/typedefs.js b/api/typedefs.js
index 24870b63ba..c8ac7a5c88 100644
--- a/api/typedefs.js
+++ b/api/typedefs.js
@@ -386,6 +386,18 @@
* @memberof typedefs
*/
+/**
+ * @exports uploadImageBuffer
+ * @typedef {import('~/server/services/Files/process').uploadImageBuffer} uploadImageBuffer
+ * @memberof typedefs
+ */
+
+/**
+ * @exports processFileURL
+ * @typedef {import('~/server/services/Files/process').processFileURL} processFileURL
+ * @memberof typedefs
+ */
+
/**
* @exports AssistantCreateParams
* @typedef {import('librechat-data-provider').AssistantCreateParams} AssistantCreateParams
diff --git a/docs/install/installation/docker_compose_install.md b/docs/install/installation/docker_compose_install.md
index 5b72f1cc62..9a14b48379 100644
--- a/docs/install/installation/docker_compose_install.md
+++ b/docs/install/installation/docker_compose_install.md
@@ -51,7 +51,12 @@ Once you have completed all the setup, you can start the LibreChat application b
That's it! If you need more detailed information on configuring your compose file, see my notes below.
## Updating LibreChat
-The following commands will fetch the latest code of LibreChat and build a new docker image.
+
+As of v0.7.0+, Docker installations transitioned from building images locally to using prebuilt images [hosted on Github Container registry](https://github.com/danny-avila?tab=packages&repo_name=LibreChat).
+
+You can still build the image locally, as shown in the commented commands below. More info on building the image locally in the [Docker Compose Override Section](../configuration/docker_override.md).
+
+The following commands will fetch the latest LibreChat project changes, including any necessary changes to the docker compose files, as well as the latest prebuilt images.
```bash
# Stop the running container(s)
@@ -63,13 +68,16 @@ git pull
# Pull the latest LibreChat image (default setup)
docker compose pull
+# If building the LibreChat image Locally, build without cache (legacy setup)
+# docker compose build --no-cache
+
# Start LibreChat
docker compose up
```
-If you're having issues running the above commands, you can try a comprehensive approach:
+If you're having issues running the above commands, you can try a comprehensive approach instead:
-Prefix commands with `sudo` according to your environment permissions.
+Note: you may need to prefix commands with `sudo` according to your environment permissions.
```bash
# Stop the container (if running)