Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: retry on MALFORMED_FUNCTION_CALL for gemini [LET-4089]
jnjpng4df4c5ea2ecb
diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 773d9599b..ff7a37b22 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -31,6 +31,8 @@ logger = get_logger(__name__)class GoogleVertexClient(LLMClientBase):+ MAX_RETRIES = model_settings.gemini_max_retries+def _get_client(self):timeout_ms = int(settings.llm_request_timeout_seconds * 1000)return genai.Client(@@ -59,12 +61,49 @@ class GoogleVertexClient(LLMClientBase):Performs underlying request to llm and returns raw response."""client = self._get_client()- response = await client.aio.models.generate_content(- model=llm_config.model,- contents=request_data["contents"],- config=request_data["config"],- )- return response.model_dump()++ # Gemini 2.5 models will often return MALFORMED_FUNCTION_CALL, force a retry+ # https://github.com/googleapis/python-aiplatform/issues/4472+ retry_count = 1+ should_retry = True+ while should_retry and retry_count <= self.MAX_RETRIES:+ response = await client.aio.models.generate_content(+ model=llm_config.model,+ contents=request_data["contents"],+ config=request_data["config"],+ )+ response_data = response.model_dump()+ is_malformed_function_call = self.is_malformed_function_call(response_data)+ if is_malformed_function_call:+ logger.warning(+ f"Received FinishReason.MALFORMED_FUNCTION_CALL in response for {llm_config.model}, retrying {retry_count}/{self.MAX_RETRIES}"+ )+ # Modify the last message if it's a heartbeat to include warning about special characters+ if request_data["contents"] and len(request_data["contents"]) > 0:+ last_message = request_data["contents"][-1]+ if last_message.get("role") == "user" and last_message.get("parts"):+ for part in last_message["parts"]:+ if "text" in part:+ try:+ # Try to parse as JSON to check if it's a heartbeat+ message_json = json_loads(part["text"])+ if message_json.get("type") == "heartbeat" and "reason" in message_json:+ # Append warning to the reason+ warning = f" RETRY {retry_count}/{self.MAX_RETRIES} ***DO NOT USE SPECIAL CHARACTERS OR QUOTATIONS INSIDE FUNCTION CALL ARGUMENTS. IF YOU MUST, MAKE SURE TO ESCAPE THEM PROPERLY***"+ message_json["reason"] = message_json["reason"] + warning+ # Update the text with modified JSON+ part["text"] = json_dumps(message_json)+ logger.warning(+ f"Modified heartbeat message with special character warning for retry {retry_count}/{self.MAX_RETRIES}"+ )+ except (json.JSONDecodeError, TypeError):+ # Not a JSON message or not a heartbeat, skip modification+ pass++ should_retry = is_malformed_function_call+ retry_count += 1++ return response_data@staticmethoddef add_dummy_model_messages(messages: List[dict]) -> List[dict]:@@ -299,7 +338,6 @@ class GoogleVertexClient(LLMClientBase):}}"""-response = GenerateContentResponse(**response_data)try:choices = []@@ -517,6 +555,14 @@ class GoogleVertexClient(LLMClientBase):def is_reasoning_model(self, llm_config: LLMConfig) -> bool:return llm_config.model.startswith("gemini-2.5-flash") or llm_config.model.startswith("gemini-2.5-pro")+ def is_malformed_function_call(self, response_data: dict) -> dict:+ response = GenerateContentResponse(**response_data)+ for candidate in response.candidates:+ content = candidate.content+ if content is None or content.role is None or content.parts is None:+ return candidate.finish_reason == "MALFORMED_FUNCTION_CALL"+ return False+@trace_methoddef handle_llm_error(self, e: Exception) -> Exception:# Fallback to base implementationdiff --git a/letta/settings.py b/letta/settings.pyindex 1cf8a62e5..81b54da65 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -145,6 +145,7 @@ class ModelSettings(BaseSettings):gemini_api_key: Optional[str] = Nonegemini_base_url: str = "https://generativelanguage.googleapis.com/"gemini_force_minimum_thinking_budget: bool = False+ gemini_max_retries: int = 5# google vertexgoogle_cloud_project: Optional[str] = Nonediff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex a63c6d43e..35a50f1b4 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -752,10 +752,11 @@ def test_tool_call(response = client.agents.messages.create(agent_id=agent_state.id,messages=USER_MESSAGE_ROLL_DICE,+ request_options={"timeout_in_seconds": 300},)except Exception as e:- if "flash" in llm_config.model and "FinishReason.MALFORMED_FUNCTION_CALL" in str(e):- pytest.skip("Skipping test for flash model due to malformed function call from llm")+ # if "flash" in llm_config.model and "FinishReason.MALFORMED_FUNCTION_CALL" in str(e):+ # pytest.skip("Skipping test for flash model due to malformed function call from llm")raise eassert_tool_call_response(response.messages, llm_config=llm_config)messages_from_db = client.agents.messages.list(agent_id=agent_state.id, after=last_message[0].id)@@ -967,6 +968,7 @@ def test_step_streaming_tool_call(response = client.agents.messages.create_stream(agent_id=agent_state.id,messages=USER_MESSAGE_ROLL_DICE,+ request_options={"timeout_in_seconds": 300},)messages = accumulate_chunks(list(response))assert_tool_call_response(messages, streaming=True, llm_config=llm_config)@@ -1115,6 +1117,7 @@ def test_token_streaming_tool_call(agent_id=agent_state.id,messages=messages_to_send,stream_tokens=True,+ request_options={"timeout_in_seconds": 300},)verify_token_streaming = (llm_config.model_endpoint_type in ["anthropic", "openai", "bedrock"] and "claude-3-5-sonnet" not in llm_config.model@@ -1183,6 +1186,7 @@ def test_background_token_streaming_greeting_with_assistant_message(messages=messages_to_send,stream_tokens=True,background=True,+ request_options={"timeout_in_seconds": 300},)verify_token_streaming = (llm_config.model_endpoint_type in ["anthropic", "openai", "bedrock"] and "claude-3-5-sonnet" not in llm_config.model@@ -1418,6 +1422,7 @@ def test_async_tool_call(run = client.agents.messages.create_async(agent_id=agent_state.id,messages=USER_MESSAGE_ROLL_DICE,+ request_options={"timeout_in_seconds": 300},)run = wait_for_run_completion(client, run.id)@@ -1639,10 +1644,11 @@ def test_auto_summarize(disable_e2b_api_key: Any, client: Letta, llm_config: LLMclient.agents.messages.create(agent_id=temp_agent_state.id,messages=[MessageCreate(role="user", content=philosophical_question)],+ request_options={"timeout_in_seconds": 300},)except Exception as e:- if "flash" in llm_config.model and "FinishReason.MALFORMED_FUNCTION_CALL" in str(e):- pytest.skip("Skipping test for flash model due to malformed function call from llm")+ # if "flash" in llm_config.model and "FinishReason.MALFORMED_FUNCTION_CALL" in str(e):+ # pytest.skip("Skipping test for flash model due to malformed function call from llm")raise etemp_agent_state = client.agents.retrieve(agent_id=temp_agent_state.id)