Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🖼️ fix: correct image extraction (#3538)
Danny Avila5baa95bd9a63
diff --git a/api/app/clients/GoogleClient.js b/api/app/clients/GoogleClient.jsindex 225d73d935..2dd921c3ce 100644--- a/api/app/clients/GoogleClient.js+++ b/api/app/clients/GoogleClient.js@@ -626,11 +626,11 @@ class GoogleClient extends BaseClient {const { onProgress, abortController } = options;const streamRate = this.options.streamRate ?? Constants.DEFAULT_STREAM_RATE;const { messages: _messages, context, examples: _examples } = instances?.[0] ?? {};-+let examples;-+let clientOptions = { ...parameters, maxRetries: 2 };-+if (this.project_id) {clientOptions['authOptions'] = {credentials: {@@ -639,16 +639,16 @@ class GoogleClient extends BaseClient {projectId: this.project_id,};}-+if (!parameters) {clientOptions = { ...clientOptions, ...this.modelOptions };}-+if (this.isGenerativeModel && !this.project_id) {clientOptions.modelName = clientOptions.model;delete clientOptions.model;}-+if (_examples && _examples.length) {examples = _examples.map((ex) => {@@ -662,26 +662,26 @@ class GoogleClient extends BaseClient {};}).filter((ex) => ex);-+clientOptions.examples = examples;}-+const model = this.createLLM(clientOptions);-+let reply = '';const messages = this.isTextModel ? _payload.trim() : _messages;-+if (!this.isVisionModel && context && messages?.length > 0) {messages.unshift(new SystemMessage(context));}-+const modelName = clientOptions.modelName ?? clientOptions.model ?? '';if (modelName?.includes('1.5') && !this.project_id) {const client = model;const requestOptions = {contents: _payload,};-+if (this.options?.promptPrefix?.length) {requestOptions.systemInstruction = {parts: [@@ -691,9 +691,9 @@ class GoogleClient extends BaseClient {],};}-+requestOptions.safetySettings = _payload.safetySettings;-+const delay = modelName.includes('flash') ? 8 : 14;const result = await client.generateContentStream(requestOptions);for await (const chunk of result.stream) {@@ -706,15 +706,15 @@ class GoogleClient extends BaseClient {}return reply;}-+const stream = await model.stream(messages, {signal: abortController.signal,timeout: 7000,safetySettings: _payload.safetySettings,});-+let delay = this.options.streamRate || 8;-+if (!this.options.streamRate) {if (this.isGenerativeModel) {delay = 12;@@ -723,7 +723,7 @@ class GoogleClient extends BaseClient {delay = 5;}}-+for await (const chunk of stream) {const chunkText = chunk?.content ?? chunk;await this.generateTextStream(chunkText, onProgress, {@@ -731,7 +731,7 @@ class GoogleClient extends BaseClient {});reply += chunkText;}-+return reply;}@@ -869,17 +869,18 @@ class GoogleClient extends BaseClient {async sendCompletion(payload, opts = {}) {payload.safetySettings = this.getSafetySettings();-+let reply = '';reply = await this.getCompletion(payload, opts);return reply.trim();}-+getSafetySettings() {return [{category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',- threshold: process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',+ threshold:+ process.env.GOOGLE_SAFETY_SEXUALLY_EXPLICIT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',},{category: 'HARM_CATEGORY_HATE_SPEECH',@@ -891,7 +892,8 @@ class GoogleClient extends BaseClient {},{category: 'HARM_CATEGORY_DANGEROUS_CONTENT',- threshold: process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',+ threshold:+ process.env.GOOGLE_SAFETY_DANGEROUS_CONTENT || 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',},];}diff --git a/api/app/clients/output_parsers/addImages.js b/api/app/clients/output_parsers/addImages.jsindex ec04bcac86..7bef60259c 100644--- a/api/app/clients/output_parsers/addImages.js+++ b/api/app/clients/output_parsers/addImages.js@@ -60,10 +60,10 @@ function addImages(intermediateSteps, responseMessage) {if (!observation || !observation.includes('![')) {return;}- const observedImagePath = observation.match(/!\[.*\]\([^)]*\)/g);+ const observedImagePath = observation.match(/!\[[^(]*\]\([^)]*\)/g);if (observedImagePath && !responseMessage.text.includes(observedImagePath[0])) {- responseMessage.text += '\n' + observation;- logger.debug('[addImages] added image from intermediateSteps:', observation);+ responseMessage.text += '\n' + observedImagePath[0];+ logger.debug('[addImages] added image from intermediateSteps:', observedImagePath[0]);}});}diff --git a/api/app/clients/output_parsers/addImages.spec.js b/api/app/clients/output_parsers/addImages.spec.jsindex eb4d87d65a..7c5a04137e 100644--- a/api/app/clients/output_parsers/addImages.spec.js+++ b/api/app/clients/output_parsers/addImages.spec.js@@ -81,4 +81,62 @@ describe('addImages', () => {addImages(intermediateSteps, responseMessage);expect(responseMessage.text).toBe(`${originalText}\n${imageMarkdown}`);});++ it('should extract only image markdowns when there is text between them', () => {+ const markdownWithTextBetweenImages = `+ + Some text between images that should not be included.+ + More text that should be ignored.+ + `;+ intermediateSteps.push({ observation: markdownWithTextBetweenImages });+ addImages(intermediateSteps, responseMessage);+ expect(responseMessage.text).toBe('\n');+ });++ it('should only return the first image when multiple images are present', () => {+ const markdownWithMultipleImages = `+ + + + `;+ intermediateSteps.push({ observation: markdownWithMultipleImages });+ addImages(intermediateSteps, responseMessage);+ expect(responseMessage.text).toBe('\n');+ });++ it('should not include any text or metadata surrounding the image markdown', () => {+ const markdownWithMetadata = `+ Title: Test Document+ Author: John Doe+ + Some content after the image.+ Vector values: [0.1, 0.2, 0.3]+ `;+ intermediateSteps.push({ observation: markdownWithMetadata });+ addImages(intermediateSteps, responseMessage);+ expect(responseMessage.text).toBe('\n');+ });++ it('should handle complex markdown with multiple images and only return the first one', () => {+ const complexMarkdown = `+ # Document Title++ ## Section 1+ Here's some text with an embedded image:+ ++ ## Section 2+ More text here...+ ++ ### Subsection+ Even more content+ + `;+ intermediateSteps.push({ observation: complexMarkdown });+ addImages(intermediateSteps, responseMessage);+ expect(responseMessage.text).toBe('\n');+ });});