Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add vertex to new agent loop (#2054)
cthomasfa4b647e853d
diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex 4a1944572..9dba906d2 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -348,7 +348,7 @@ class LettaAgent(BaseAgent):results = await self._send_message_to_agents_matching_tags(**tool_args)log_event(name="finish_send_message_to_agents_matching_tags", attributes=tool_args)return json.dumps(results), True- elif target_tool.type == ToolType.EXTERNAL_COMPOSIO:+ elif target_tool.tool_type == ToolType.EXTERNAL_COMPOSIO:log_event(name=f"start_composio_{tool_name}_execution", attributes=tool_args)log_event(name=f"finish_compsio_{tool_name}_execution", attributes=tool_args)return tool_execution_result.func_return, Truediff --git a/letta/llm_api/google_ai_client.py b/letta/llm_api/google_ai_client.pyindex 925771951..c41ad0501 100644--- a/letta/llm_api/google_ai_client.py+++ b/letta/llm_api/google_ai_client.py@@ -288,7 +288,7 @@ class GoogleAIClient(LLMClientBase):# Per https://ai.google.dev/gemini-api/docs/function-calling?example=meeting#notes_and_limitations# * Only a subset of the OpenAPI schema is supported.# * Supported parameter types in Python are limited.- unsupported_keys = ["default", "exclusiveMaximum", "exclusiveMinimum"]+ unsupported_keys = ["default", "exclusiveMaximum", "exclusiveMinimum", "additionalProperties"]keys_to_remove_at_this_level = [key for key in unsupported_keys if key in schema_part]for key_to_remove in keys_to_remove_at_this_level:logger.warning(f"Removing unsupported keyword '{key_to_remove}' from schema part.")diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 4edd03b27..9fd66558b 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -5,14 +5,14 @@ from google import genaifrom google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, GenerateContentResponse, ThinkingConfig, ToolConfigfrom letta.helpers.datetime_helpers import get_utc_time_int-from letta.helpers.json_helpers import json_dumps+from letta.helpers.json_helpers import json_dumps, json_loadsfrom letta.llm_api.google_ai_client import GoogleAIClientfrom letta.local_llm.json_parser import clean_json_string_extra_backslashfrom letta.local_llm.utils import count_tokensfrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, FunctionCall, Message, ToolCall, UsageStatistics-from letta.settings import model_settings+from letta.settings import model_settings, settingsfrom letta.utils import get_tool_call_id@@ -35,6 +35,23 @@ class GoogleVertexClient(GoogleAIClient):)return response.model_dump()+ async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:+ """+ Performs underlying request to llm and returns raw response.+ """+ client = genai.Client(+ vertexai=True,+ project=model_settings.google_cloud_project,+ location=model_settings.google_cloud_location,+ http_options={"api_version": "v1"},+ )+ response = await client.aio.models.generate_content(+ model=llm_config.model,+ contents=request_data["contents"],+ config=request_data["config"],+ )+ return response.model_dump()+def build_request_data(self,messages: List[PydanticMessage],@@ -49,16 +66,21 @@ class GoogleVertexClient(GoogleAIClient):request_data["config"] = request_data.pop("generation_config")request_data["config"]["tools"] = request_data.pop("tools")- tool_names = [t["name"] for t in tools]- tool_config = ToolConfig(- function_calling_config=FunctionCallingConfig(- # ANY mode forces the model to predict only function calls- mode=FunctionCallingConfigMode.ANY,- # Provide the list of tools (though empty should also work, it seems not to)- allowed_function_names=tool_names,+ tool_names = [t["name"] for t in tools] if tools else []+ if len(tool_names) == 1 and settings.use_vertex_structured_outputs_experimental:+ request_data["config"]["response_mime_type"] = "application/json"+ request_data["config"]["response_schema"] = self.get_function_call_response_schema(tools[0])+ del request_data["config"]["tools"]+ else:+ tool_config = ToolConfig(+ function_calling_config=FunctionCallingConfig(+ # ANY mode forces the model to predict only function calls+ mode=FunctionCallingConfigMode.ANY,+ # Provide the list of tools (though empty should also work, it seems not to)+ allowed_function_names=tool_names,+ ))- )- request_data["config"]["tool_config"] = tool_config.model_dump()+ request_data["config"]["tool_config"] = tool_config.model_dump()# Add thinking_config# If enable_reasoner is False, set thinking_budget to 0@@ -110,12 +132,16 @@ class GoogleVertexClient(GoogleAIClient):for candidate in response.candidates:content = candidate.content- # if "role" not in content or not content["role"]:- # # This means the response is malformed like MALFORMED_FUNCTION_CALL- # # NOTE: must be a ValueError to trigger a retry- # raise ValueError(f"Error in response data from LLM: {response_data}")- # role = content["role"]- # assert role == "model", f"Unknown role in response: {role}"+ if content.role is None or content.parts is None:+ # This means the response is malformed like MALFORMED_FUNCTION_CALL+ # NOTE: must be a ValueError to trigger a retry+ if candidate.finish_reason == "MALFORMED_FUNCTION_CALL":+ raise ValueError(f"Error in response data from LLM: {candidate.finish_message[:350]}...")+ else:+ raise ValueError(f"Error in response data from LLM: {response_data}")++ role = content.role+ assert role == "model", f"Unknown role in response: {role}"parts = content.parts@@ -167,15 +193,50 @@ class GoogleVertexClient(GoogleAIClient):)else:+ try:+ # Structured output tool call+ function_call = json_loads(response_message.text)+ function_name = function_call["name"]+ function_args = function_call["args"]+ assert isinstance(function_args, dict), function_args- # Inner thoughts are the content by default- inner_thoughts = response_message.text+ # NOTE: this also involves stripping the inner monologue out of the function+ if llm_config.put_inner_thoughts_in_kwargs:+ from letta.local_llm.constants import INNER_THOUGHTS_KWARG- # Google AI API doesn't generate tool call IDs- openai_response_message = Message(- role="assistant", # NOTE: "model" -> "assistant"- content=inner_thoughts,- )+ assert (+ INNER_THOUGHTS_KWARG in function_args+ ), f"Couldn't find inner thoughts in function args:\n{function_call}"+ inner_thoughts = function_args.pop(INNER_THOUGHTS_KWARG)+ assert inner_thoughts is not None, f"Expected non-null inner thoughts function arg:\n{function_call}"+ else:+ inner_thoughts = None++ # Google AI API doesn't generate tool call IDs+ openai_response_message = Message(+ role="assistant", # NOTE: "model" -> "assistant"+ content=inner_thoughts,+ tool_calls=[+ ToolCall(+ id=get_tool_call_id(),+ type="function",+ function=FunctionCall(+ name=function_name,+ arguments=clean_json_string_extra_backslash(json_dumps(function_args)),+ ),+ )+ ],+ )++ except:+ # Inner thoughts are the content by default+ inner_thoughts = response_message.text++ # Google AI API doesn't generate tool call IDs+ openai_response_message = Message(+ role="assistant", # NOTE: "model" -> "assistant"+ content=inner_thoughts,+ )# Google AI API uses different finish reason strings than OpenAI# OpenAI: 'stop', 'length', 'function_call', 'content_filter', null@@ -244,3 +305,17 @@ class GoogleVertexClient(GoogleAIClient):)except KeyError as e:raise e++ def get_function_call_response_schema(self, tool: dict) -> dict:+ return {+ "type": "OBJECT",+ "properties": {+ "name": {"type": "STRING", "enum": [tool["name"]]},+ "args": {+ "type": "OBJECT",+ "properties": tool["parameters"]["properties"],+ "required": tool["parameters"]["required"],+ },+ },+ "required": ["name", "args"],+ }diff --git a/letta/settings.py b/letta/settings.pyindex 1787a8b0d..a3149ac9b 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -195,6 +195,7 @@ class Settings(BaseSettings):# experimental toggleuse_experimental: bool = False+ use_vertex_structured_outputs_experimental: bool = False# LLM provider client settingshttpx_max_retries: int = 5diff --git a/tests/integration_test_experimental.py b/tests/integration_test_experimental.pyindex e690023cd..0b9df3893 100644--- a/tests/integration_test_experimental.py+++ b/tests/integration_test_experimental.py@@ -24,7 +24,7 @@ from letta.services.message_manager import MessageManagerfrom letta.services.passage_manager import PassageManagerfrom letta.services.tool_manager import ToolManagerfrom letta.services.user_manager import UserManager-from letta.settings import model_settings+from letta.settings import model_settings, settings# --- Server Management --- #@@ -263,7 +263,55 @@ async def test_rethink_tool(disable_e2b_api_key, openai_client, agent_state, mesassert "chicken" not in AgentManager().get_agent_by_id(agent_state.id, actor).memory.get_block("human").valueresponse = await agent.step([LettaMessageCreate(role="user", content=[LettaTextContent(text=message)])])- assert "chicken" in AgentManager().get_agent_by_id(agent_state.id, actor).memory.get_block("human").value+ assert "chicken" in AgentManager().get_agent_by_id(agent_state.id, actor).memory.get_block("human").value.lower()+++async def test_vertex_send_message_structured_outputs(disable_e2b_api_key, client):+ original_experimental_key = settings.use_vertex_structured_outputs_experimental+ settings.use_vertex_structured_outputs_experimental = True+ try:+ actor = UserManager().get_user_or_default(user_id="asf")++ stale_agents = AgentManager().list_agents(actor=actor, limit=300)+ for agent in stale_agents:+ AgentManager().delete_agent(agent_id=agent.id, actor=actor)++ manager_agent_state = client.agents.create(+ name=f"manager",+ include_base_tools=False, # change this to True to repro MALFORMED FUNCTION CALL error+ tools=["send_message"],+ tags=["manager"],+ model="google_vertex/gemini-2.5-flash-preview-04-17",+ embedding="letta/letta-free",+ )+ manager_agent = LettaAgent(+ agent_id=manager_agent_state.id,+ message_manager=MessageManager(),+ agent_manager=AgentManager(),+ block_manager=BlockManager(),+ passage_manager=PassageManager(),+ actor=actor,+ )++ response = await manager_agent.step(+ [+ LettaMessageCreate(+ role="user",+ content=[+ LettaTextContent(text=("Check the weather in Seattle.")),+ ],+ ),+ ]+ )+ assert len(response.messages) == 3+ assert response.messages[0].message_type == "user_message"+ # Shouldn't this have reasoning message?+ # assert response.messages[1].message_type == "reasoning_message"+ assert response.messages[1].message_type == "assistant_message"+ assert response.messages[2].message_type == "tool_return_message"+ finally:+ settings.use_vertex_structured_outputs_experimental = original_experimental_key@pytest.mark.asyncio@@ -304,7 +352,6 @@ async def test_multi_agent_broadcast(disable_e2b_api_key, client, openai_client,embedding="letta/letta-free",),)-response = await manager_agent.step([LettaMessageCreate(