Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add streaming support to vertex client (#5106)
cthomas6b6b82a07a4b
diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 180b38ab3..123d46ee8 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -1,6 +1,6 @@import jsonimport uuid-from typing import List, Optional+from typing import AsyncIterator, List, Optionalfrom google import genaifrom google.genai import errors@@ -138,6 +138,15 @@ class GoogleVertexClient(LLMClientBase):raise RuntimeError("Failed to get response data after all retries")return response_data+ @trace_method+ async def stream_async(self, request_data: dict, llm_config: LLMConfig) -> AsyncIterator[GenerateContentResponse]:+ client = self._get_client()+ return await client.aio.models.generate_content_stream(+ model=llm_config.model,+ contents=request_data["contents"],+ config=request_data["config"],+ )+@staticmethoddef add_dummy_model_messages(messages: List[dict]) -> List[dict]:"""Google AI API requires all function call returns are immediately followed by a 'model' role message.