The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
Feat/llm monitoring callback (#2877)
John Lockwood4c748423fc8c
diff --git a/docs/components/llms/config.mdx b/docs/components/llms/config.mdxindex 9399d0e7a..5a7ca6459 100644--- a/docs/components/llms/config.mdx+++ b/docs/components/llms/config.mdx@@ -119,6 +119,7 @@ Here's a comprehensive list of all parameters that can be used across different| `seed` | Seed for deterministic sampling | Sarvam || `stop` | Stop sequences (max 4) | Sarvam || `lmstudio_base_url` | Base URL for LM Studio API | LM Studio |+ | `response_callback` | LLM response callback function | OpenAI |</Tab><Tab title="TypeScript">| Parameter | Description | Provider |diff --git a/mem0/configs/llms/openai.py b/mem0/configs/llms/openai.pyindex 459557e58..960dc0f23 100644--- a/mem0/configs/llms/openai.py+++ b/mem0/configs/llms/openai.py@@ -1,4 +1,4 @@-from typing import List, Optional+from typing import Any, Callable, List, Optionalfrom mem0.configs.llms.base import BaseLlmConfig@@ -28,6 +28,8 @@ class OpenAIConfig(BaseLlmConfig):openrouter_base_url: Optional[str] = None,site_url: Optional[str] = None,app_name: Optional[str] = None,+ # Response monitoring callback+ response_callback: Optional[Callable[[Any, dict, dict], None]] = None,):"""Initialize OpenAI configuration.@@ -48,6 +50,7 @@ class OpenAIConfig(BaseLlmConfig):openrouter_base_url: OpenRouter base URL, defaults to Nonesite_url: Site URL for OpenRouter, defaults to Noneapp_name: Application name for OpenRouter, defaults to None+ response_callback: Optional callback for monitoring LLM responses."""# Initialize base parameterssuper().__init__(@@ -69,3 +72,5 @@ class OpenAIConfig(BaseLlmConfig):self.openrouter_base_url = openrouter_base_urlself.site_url = site_urlself.app_name = app_name+ # Response monitoring+ self.response_callback = response_callbackdiff --git a/mem0/llms/openai.py b/mem0/llms/openai.pyindex 58d066d06..e3cb917db 100644--- a/mem0/llms/openai.py+++ b/mem0/llms/openai.py@@ -1,4 +1,5 @@import json+import loggingimport osfrom typing import Dict, List, Optional, Union@@ -130,6 +131,13 @@ class OpenAILLM(LLMBase):if tools: # TODO: Remove tools if no issues found with new memory addition logicparams["tools"] = toolsparams["tool_choice"] = tool_choice-response = self.client.chat.completions.create(**params)- return self._parse_response(response, tools)+ parsed_response = self._parse_response(response, tools)+ if self.config.response_callback:+ try:+ self.config.response_callback(self, response, params)+ except Exception as e:+ # Log error but don't propagate+ logging.error(f"Error due to callback: {e}")+ pass+ return parsed_responsediff --git a/tests/llms/test_openai.py b/tests/llms/test_openai.pyindex 265213b97..f5b9abc3a 100644--- a/tests/llms/test_openai.py+++ b/tests/llms/test_openai.py@@ -104,3 +104,106 @@ def test_generate_response_with_tools(mock_openai_client):assert len(response["tool_calls"]) == 1assert response["tool_calls"][0]["name"] == "add_memory"assert response["tool_calls"][0]["arguments"] == {"data": "Today is a sunny day."}+++def test_response_callback_invocation(mock_openai_client):+ # Setup mock callback+ mock_callback = Mock()++ config = OpenAIConfig(model="gpt-4o", response_callback=mock_callback)+ llm = OpenAILLM(config)+ messages = [{"role": "user", "content": "Test callback"}]++ # Mock response+ mock_response = Mock()+ mock_response.choices = [Mock(message=Mock(content="Response"))]+ mock_openai_client.chat.completions.create.return_value = mock_response++ # Call method+ llm.generate_response(messages)++ # Verify callback called with correct arguments+ mock_callback.assert_called_once()+ args = mock_callback.call_args[0]+ assert args[0] is llm # llm_instance+ assert args[1] == mock_response # raw_response+ assert "messages" in args[2] # params+++def test_no_response_callback(mock_openai_client):+ config = OpenAIConfig(model="gpt-4o")+ llm = OpenAILLM(config)+ messages = [{"role": "user", "content": "Test no callback"}]++ # Mock response+ mock_response = Mock()+ mock_response.choices = [Mock(message=Mock(content="Response"))]+ mock_openai_client.chat.completions.create.return_value = mock_response++ # Should complete without calling any callback+ response = llm.generate_response(messages)+ assert response == "Response"++ # Verify no callback is set+ assert llm.config.response_callback is None+++def test_callback_exception_handling(mock_openai_client):+ # Callback that raises exception+ def faulty_callback(*args):+ raise ValueError("Callback error")++ config = OpenAIConfig(model="gpt-4o", response_callback=faulty_callback)+ llm = OpenAILLM(config)+ messages = [{"role": "user", "content": "Test exception"}]++ # Mock response+ mock_response = Mock()+ mock_response.choices = [Mock(message=Mock(content="Expected response"))]+ mock_openai_client.chat.completions.create.return_value = mock_response++ # Should complete without raising+ response = llm.generate_response(messages)+ assert response == "Expected response"++ # Verify callback was called (even though it raised an exception)+ assert llm.config.response_callback is faulty_callback+++def test_callback_with_tools(mock_openai_client):+ mock_callback = Mock()+ config = OpenAIConfig(model="gpt-4o", response_callback=mock_callback)+ llm = OpenAILLM(config)+ messages = [{"role": "user", "content": "Test tools"}]+ tools = [+ {+ "type": "function",+ "function": {+ "name": "test_tool",+ "description": "A test tool",+ "parameters": {+ "type": "object",+ "properties": {"param1": {"type": "string"}},+ "required": ["param1"],+ },+ }+ }+ ]++ # Mock tool response+ mock_response = Mock()+ mock_message = Mock()+ mock_message.content = "Tool response"+ mock_tool_call = Mock()+ mock_tool_call.function.name = "test_tool"+ mock_tool_call.function.arguments = '{"param1": "value1"}'+ mock_message.tool_calls = [mock_tool_call]+ mock_response.choices = [Mock(message=mock_message)]+ mock_openai_client.chat.completions.create.return_value = mock_response++ llm.generate_response(messages, tools=tools)++ # Verify callback called with tool response+ mock_callback.assert_called_once()+ # Check that tool_calls exists in the message+ assert hasattr(mock_callback.call_args[0][1].choices[0].message, 'tool_calls')