Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

✂️ fix: Remove Image Payloads from Memory Processing (#8770)

Danny Avila6fc9abd4ad68
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index f9dacbe5ad..d8fda5159a 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -512,6 +512,39 @@ class AgentClient extends BaseClient {
return withoutKeys;
}
+ /**
+ * Filters out image URLs from message content
+ * @param {BaseMessage} message - The message to filter
+ * @returns {BaseMessage} - A new message with image URLs removed
+ */
+ filterImageUrls(message) {
+ if (!message.content || typeof message.content === 'string') {
+ return message;
+ }
+
+ if (Array.isArray(message.content)) {
+ const filteredContent = message.content.filter(
+ (part) => part.type !== ContentTypes.IMAGE_URL,
+ );
+
+ if (filteredContent.length === 1 && filteredContent[0].type === ContentTypes.TEXT) {
+ const MessageClass = message.constructor;
+ return new MessageClass({
+ content: filteredContent[0].text,
+ additional_kwargs: message.additional_kwargs,
+ });
+ }
+
+ const MessageClass = message.constructor;
+ return new MessageClass({
+ content: filteredContent,
+ additional_kwargs: message.additional_kwargs,
+ });
+ }
+
+ return message;
+ }
+
/**
* @param {BaseMessage[]} messages
* @returns {Promise<void | (TAttachment | null)[]>}
@@ -540,7 +573,8 @@ class AgentClient extends BaseClient {
}
}
- const bufferString = getBufferString(messagesToProcess);
+ const filteredMessages = messagesToProcess.map((msg) => this.filterImageUrls(msg));
+ const bufferString = getBufferString(filteredMessages);
const bufferMessage = new HumanMessage(`# Current Chat:\n\n${bufferString}`);
return await this.processMemory([bufferMessage]);
} catch (error) {
diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js
index aab972fecb..92b97ad006 100644
--- a/api/server/controllers/agents/client.test.js
+++ b/api/server/controllers/agents/client.test.js
@@ -727,4 +727,231 @@ describe('AgentClient - titleConvo', () => {
});
});
});
+
+ describe('runMemory method', () => {
+ let client;
+ let mockReq;
+ let mockRes;
+ let mockAgent;
+ let mockOptions;
+ let mockProcessMemory;
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+
+ mockAgent = {
+ id: 'agent-123',
+ endpoint: EModelEndpoint.openAI,
+ provider: EModelEndpoint.openAI,
+ model_parameters: {
+ model: 'gpt-4',
+ },
+ };
+
+ mockReq = {
+ app: {
+ locals: {
+ memory: {
+ messageWindowSize: 3,
+ },
+ },
+ },
+ user: {
+ id: 'user-123',
+ personalization: {
+ memories: true,
+ },
+ },
+ };
+
+ mockRes = {};
+
+ mockOptions = {
+ req: mockReq,
+ res: mockRes,
+ agent: mockAgent,
+ };
+
+ mockProcessMemory = jest.fn().mockResolvedValue([]);
+
+ client = new AgentClient(mockOptions);
+ client.processMemory = mockProcessMemory;
+ client.conversationId = 'convo-123';
+ client.responseMessageId = 'response-123';
+ });
+
+ it('should filter out image URLs from message content', async () => {
+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+ const messages = [
+ new HumanMessage({
+ content: [
+ {
+ type: 'text',
+ text: 'What is in this image?',
+ },
+ {
+ type: 'image_url',
+ image_url: {
+ url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+ detail: 'auto',
+ },
+ },
+ ],
+ }),
+ new AIMessage('I can see a small red pixel in the image.'),
+ new HumanMessage({
+ content: [
+ {
+ type: 'text',
+ text: 'What about this one?',
+ },
+ {
+ type: 'image_url',
+ image_url: {
+ url: 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/',
+ detail: 'high',
+ },
+ },
+ ],
+ }),
+ ];
+
+ await client.runMemory(messages);
+
+ expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+ // Verify the buffer message was created
+ expect(processedMessage.constructor.name).toBe('HumanMessage');
+ expect(processedMessage.content).toContain('# Current Chat:');
+
+ // Verify that image URLs are not in the buffer string
+ expect(processedMessage.content).not.toContain('image_url');
+ expect(processedMessage.content).not.toContain('data:image');
+ expect(processedMessage.content).not.toContain('base64');
+
+ // Verify text content is preserved
+ expect(processedMessage.content).toContain('What is in this image?');
+ expect(processedMessage.content).toContain('I can see a small red pixel in the image.');
+ expect(processedMessage.content).toContain('What about this one?');
+ });
+
+ it('should handle messages with only text content', async () => {
+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+ const messages = [
+ new HumanMessage('Hello, how are you?'),
+ new AIMessage('I am doing well, thank you!'),
+ new HumanMessage('That is great to hear.'),
+ ];
+
+ await client.runMemory(messages);
+
+ expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+ expect(processedMessage.content).toContain('Hello, how are you?');
+ expect(processedMessage.content).toContain('I am doing well, thank you!');
+ expect(processedMessage.content).toContain('That is great to hear.');
+ });
+
+ it('should handle mixed content types correctly', async () => {
+ const { HumanMessage } = require('@langchain/core/messages');
+ const { ContentTypes } = require('librechat-data-provider');
+
+ const messages = [
+ new HumanMessage({
+ content: [
+ {
+ type: 'text',
+ text: 'Here is some text',
+ },
+ {
+ type: ContentTypes.IMAGE_URL,
+ image_url: {
+ url: 'https://example.com/image.png',
+ },
+ },
+ {
+ type: 'text',
+ text: ' and more text',
+ },
+ ],
+ }),
+ ];
+
+ await client.runMemory(messages);
+
+ expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+ // Should contain text parts but not image URLs
+ expect(processedMessage.content).toContain('Here is some text');
+ expect(processedMessage.content).toContain('and more text');
+ expect(processedMessage.content).not.toContain('example.com/image.png');
+ expect(processedMessage.content).not.toContain('IMAGE_URL');
+ });
+
+ it('should preserve original messages without mutation', async () => {
+ const { HumanMessage } = require('@langchain/core/messages');
+ const originalContent = [
+ {
+ type: 'text',
+ text: 'Original text',
+ },
+ {
+ type: 'image_url',
+ image_url: {
+ url: 'data:image/png;base64,ABC123',
+ },
+ },
+ ];
+
+ const messages = [
+ new HumanMessage({
+ content: [...originalContent],
+ }),
+ ];
+
+ await client.runMemory(messages);
+
+ // Verify original message wasn't mutated
+ expect(messages[0].content).toHaveLength(2);
+ expect(messages[0].content[1].type).toBe('image_url');
+ expect(messages[0].content[1].image_url.url).toBe('data:image/png;base64,ABC123');
+ });
+
+ it('should handle message window size correctly', async () => {
+ const { HumanMessage, AIMessage } = require('@langchain/core/messages');
+ const messages = [
+ new HumanMessage('Message 1'),
+ new AIMessage('Response 1'),
+ new HumanMessage('Message 2'),
+ new AIMessage('Response 2'),
+ new HumanMessage('Message 3'),
+ new AIMessage('Response 3'),
+ ];
+
+ // Window size is set to 3 in mockReq
+ await client.runMemory(messages);
+
+ expect(mockProcessMemory).toHaveBeenCalledTimes(1);
+ const processedMessage = mockProcessMemory.mock.calls[0][0][0];
+
+ // Should only include last 3 messages due to window size
+ expect(processedMessage.content).toContain('Message 3');
+ expect(processedMessage.content).toContain('Response 3');
+ expect(processedMessage.content).not.toContain('Message 1');
+ expect(processedMessage.content).not.toContain('Response 1');
+ });
+
+ it('should return early if processMemory is not set', async () => {
+ const { HumanMessage } = require('@langchain/core/messages');
+ client.processMemory = null;
+
+ const result = await client.runMemory([new HumanMessage('Test')]);
+
+ expect(result).toBeUndefined();
+ expect(mockProcessMemory).not.toHaveBeenCalled();
+ });
+ });
});