Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
✂️ fix: Remove Image Payloads from Memory Processing (#8770)
Danny Avila6fc9abd4ad68
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex f9dacbe5ad..d8fda5159a 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -512,6 +512,39 @@ class AgentClient extends BaseClient {return withoutKeys;}+ /**+ * Filters out image URLs from message content+ * @param {BaseMessage} message - The message to filter+ * @returns {BaseMessage} - A new message with image URLs removed+ */+ filterImageUrls(message) {+ if (!message.content || typeof message.content === 'string') {+ return message;+ }++ if (Array.isArray(message.content)) {+ const filteredContent = message.content.filter(+ (part) => part.type !== ContentTypes.IMAGE_URL,+ );++ if (filteredContent.length === 1 && filteredContent[0].type === ContentTypes.TEXT) {+ const MessageClass = message.constructor;+ return new MessageClass({+ content: filteredContent[0].text,+ additional_kwargs: message.additional_kwargs,+ });+ }++ const MessageClass = message.constructor;+ return new MessageClass({+ content: filteredContent,+ additional_kwargs: message.additional_kwargs,+ });+ }++ return message;+ }+/*** @param {BaseMessage[]} messages* @returns {Promise<void | (TAttachment | null)[]>}@@ -540,7 +573,8 @@ class AgentClient extends BaseClient {}}- const bufferString = getBufferString(messagesToProcess);+ const filteredMessages = messagesToProcess.map((msg) => this.filterImageUrls(msg));+ const bufferString = getBufferString(filteredMessages);const bufferMessage = new HumanMessage(`# Current Chat:\n\n${bufferString}`);return await this.processMemory([bufferMessage]);} catch (error) {diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.jsindex aab972fecb..92b97ad006 100644--- a/api/server/controllers/agents/client.test.js+++ b/api/server/controllers/agents/client.test.js@@ -727,4 +727,231 @@ describe('AgentClient - titleConvo', () => {});});});++ describe('runMemory method', () => {+ let client;+ let mockReq;+ let mockRes;+ let mockAgent;+ let mockOptions;+ let mockProcessMemory;++ beforeEach(() => {+ jest.clearAllMocks();++ mockAgent = {+ id: 'agent-123',+ endpoint: EModelEndpoint.openAI,+ provider: EModelEndpoint.openAI,+ model_parameters: {+ model: 'gpt-4',+ },+ };++ mockReq = {+ app: {+ locals: {+ memory: {+ messageWindowSize: 3,+ },+ },+ },+ user: {+ id: 'user-123',+ personalization: {+ memories: true,+ },+ },+ };++ mockRes = {};++ mockOptions = {+ req: mockReq,+ res: mockRes,+ agent: mockAgent,+ };++ mockProcessMemory = jest.fn().mockResolvedValue([]);++ client = new AgentClient(mockOptions);+ client.processMemory = mockProcessMemory;+ client.conversationId = 'convo-123';+ client.responseMessageId = 'response-123';+ });++ it('should filter out image URLs from message content', async () => {+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');+ const messages = [+ new HumanMessage({+ content: [+ {+ type: 'text',+ text: 'What is in this image?',+ },+ {+ type: 'image_url',+ image_url: {+ url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',+ detail: 'auto',+ },+ },+ ],+ }),+ new AIMessage('I can see a small red pixel in the image.'),+ new HumanMessage({+ content: [+ {+ type: 'text',+ text: 'What about this one?',+ },+ {+ type: 'image_url',+ image_url: {+ url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/',+ detail: 'high',+ },+ },+ ],+ }),+ ];++ await client.runMemory(messages);++ expect(mockProcessMemory).toHaveBeenCalledTimes(1);+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];++ // Verify the buffer message was created+ expect(processedMessage.constructor.name).toBe('HumanMessage');+ expect(processedMessage.content).toContain('# Current Chat:');++ // Verify that image URLs are not in the buffer string+ expect(processedMessage.content).not.toContain('image_url');+ expect(processedMessage.content).not.toContain('data:image');+ expect(processedMessage.content).not.toContain('base64');++ // Verify text content is preserved+ expect(processedMessage.content).toContain('What is in this image?');+ expect(processedMessage.content).toContain('I can see a small red pixel in the image.');+ expect(processedMessage.content).toContain('What about this one?');+ });++ it('should handle messages with only text content', async () => {+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');+ const messages = [+ new HumanMessage('Hello, how are you?'),+ new AIMessage('I am doing well, thank you!'),+ new HumanMessage('That is great to hear.'),+ ];++ await client.runMemory(messages);++ expect(mockProcessMemory).toHaveBeenCalledTimes(1);+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];++ expect(processedMessage.content).toContain('Hello, how are you?');+ expect(processedMessage.content).toContain('I am doing well, thank you!');+ expect(processedMessage.content).toContain('That is great to hear.');+ });++ it('should handle mixed content types correctly', async () => {+ const { HumanMessage } = require('@langchain/core/messages');+ const { ContentTypes } = require('librechat-data-provider');++ const messages = [+ new HumanMessage({+ content: [+ {+ type: 'text',+ text: 'Here is some text',+ },+ {+ type: ContentTypes.IMAGE_URL,+ image_url: {+ url: 'https://example.com/image.png',+ },+ },+ {+ type: 'text',+ text: ' and more text',+ },+ ],+ }),+ ];++ await client.runMemory(messages);++ expect(mockProcessMemory).toHaveBeenCalledTimes(1);+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];++ // Should contain text parts but not image URLs+ expect(processedMessage.content).toContain('Here is some text');+ expect(processedMessage.content).toContain('and more text');+ expect(processedMessage.content).not.toContain('example.com/image.png');+ expect(processedMessage.content).not.toContain('IMAGE_URL');+ });++ it('should preserve original messages without mutation', async () => {+ const { HumanMessage } = require('@langchain/core/messages');+ const originalContent = [+ {+ type: 'text',+ text: 'Original text',+ },+ {+ type: 'image_url',+ image_url: {+ url: 'data:image/png;base64,ABC123',+ },+ },+ ];++ const messages = [+ new HumanMessage({+ content: [...originalContent],+ }),+ ];++ await client.runMemory(messages);++ // Verify original message wasn't mutated+ expect(messages[0].content).toHaveLength(2);+ expect(messages[0].content[1].type).toBe('image_url');+ expect(messages[0].content[1].image_url.url).toBe('data:image/png;base64,ABC123');+ });++ it('should handle message window size correctly', async () => {+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');+ const messages = [+ new HumanMessage('Message 1'),+ new AIMessage('Response 1'),+ new HumanMessage('Message 2'),+ new AIMessage('Response 2'),+ new HumanMessage('Message 3'),+ new AIMessage('Response 3'),+ ];++ // Window size is set to 3 in mockReq+ await client.runMemory(messages);++ expect(mockProcessMemory).toHaveBeenCalledTimes(1);+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];++ // Should only include last 3 messages due to window size+ expect(processedMessage.content).toContain('Message 3');+ expect(processedMessage.content).toContain('Response 3');+ expect(processedMessage.content).not.toContain('Message 1');+ expect(processedMessage.content).not.toContain('Response 1');+ });++ it('should return early if processMemory is not set', async () => {+ const { HumanMessage } = require('@langchain/core/messages');+ client.processMemory = null;++ const result = await client.runMemory([new HumanMessage('Test')]);++ expect(result).toBeUndefined();+ expect(mockProcessMemory).not.toHaveBeenCalled();+ });+ });});