Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: support multi content part input (#2717)
cthomasb332ebfa858e
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.pyindex 1f61d960a..24c66bbff 100644--- a/letta/llm_api/anthropic_client.py+++ b/letta/llm_api/anthropic_client.py@@ -29,7 +29,6 @@ from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARGfrom letta.log import get_loggerfrom letta.otel.tracing import trace_methodfrom letta.schemas.enums import ProviderCategory-from letta.schemas.letta_message_content import MessageContentTypefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.openai.chat_completion_request import Tool as OpenAITool@@ -252,8 +251,6 @@ class AnthropicClient(LLMClientBase):for m in messages[1:]]- data["messages"] = fill_image_content_in_messages(data["messages"], messages)-# Ensure first message is userif data["messages"][0]["role"] != "user":data["messages"] = [{"role": "user", "content": DUMMY_FIRST_USER_MESSAGE}] + data["messages"]@@ -656,52 +653,3 @@ def strip_xml_tags_streaming(string: str, tag: Optional[str]) -> str:result = result.replace(part, "")return result---def fill_image_content_in_messages(anthropic_message_list: List[dict], pydantic_message_list: List[PydanticMessage]) -> List[dict]:- """- Converts image content to anthropic format.- """-- if len(anthropic_message_list) != len(pydantic_message_list):- return anthropic_message_list-- new_message_list = []- for idx in range(len(anthropic_message_list)):- anthropic_message, pydantic_message = anthropic_message_list[idx], pydantic_message_list[idx]- if pydantic_message.role != "user":- new_message_list.append(anthropic_message)- continue-- if not isinstance(pydantic_message.content, list) or (- len(pydantic_message.content) == 1 and pydantic_message.content.type == MessageContentType.text- ):- new_message_list.append(anthropic_message)- continue-- message_content = []- for content in pydantic_message.content:- if content.type == MessageContentType.text:- message_content.append(- {- "type": "input_text",- "text": content.text,- }- )- elif content.type == MessageContentType.image:- message_content.append(- {- "type": "image",- "source": {- "type": content.source.type,- "media_type": content.source.media_type,- "data": content.source.data,- },- }- )- else:- raise ValueError(f"Unsupported content type {content.type}")-- new_message_list.append({"role": "user", "input": message_content})-- return new_message_listdiff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.pyindex 5738a6b3c..25b1e00fa 100644--- a/letta/llm_api/google_vertex_client.py+++ b/letta/llm_api/google_vertex_client.py@@ -3,7 +3,7 @@ import uuidfrom typing import List, Optionalfrom google import genai-from google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, GenerateContentResponse, Part, ThinkingConfig, ToolConfig+from google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, GenerateContentResponse, ThinkingConfig, ToolConfigfrom letta.constants import NON_USER_MSG_PREFIXfrom letta.helpers.datetime_helpers import get_utc_time_int@@ -13,7 +13,6 @@ from letta.local_llm.json_parser import clean_json_string_extra_backslashfrom letta.local_llm.utils import count_tokensfrom letta.log import get_loggerfrom letta.otel.tracing import trace_method-from letta.schemas.letta_message_content import MessageContentTypefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.openai.chat_completion_request import Tool@@ -219,7 +218,7 @@ class GoogleVertexClient(LLMClientBase):)request_data = {- "contents": fill_image_content_in_messages(contents, messages),+ "contents": contents,"config": {"temperature": llm_config.temperature,"max_output_tokens": llm_config.max_tokens,@@ -485,43 +484,3 @@ class GoogleVertexClient(LLMClientBase):"propertyOrdering": ["name", "args"],"required": ["name", "args"],}---def fill_image_content_in_messages(google_ai_message_list: List[dict], pydantic_message_list: List[PydanticMessage]) -> List[dict]:- """- Converts image content to openai format.- """-- if len(google_ai_message_list) != len(pydantic_message_list):- return google_ai_message_list-- new_message_list = []- for idx in range(len(google_ai_message_list)):- google_ai_message, pydantic_message = google_ai_message_list[idx], pydantic_message_list[idx]- if pydantic_message.role != "user":- new_message_list.append(google_ai_message)- continue-- if not isinstance(pydantic_message.content, list) or (- len(pydantic_message.content) == 1 and pydantic_message.content[0].type == MessageContentType.text- ):- new_message_list.append(google_ai_message)- continue-- message_content = []- for content in pydantic_message.content:- if content.type == MessageContentType.text:- message_content.append({"text": content.text})- elif content.type == MessageContentType.image:- message_content.append(- Part.from_bytes(- mime_type=content.source.media_type,- data=content.source.data,- )- )- else:- raise ValueError(f"Unsupported content type {content.type}")-- new_message_list.append({"role": "user", "input": message_content})-- return new_message_listdiff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex 56ca5f427..c859e50a7 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -432,7 +432,7 @@ def fill_image_content_in_messages(openai_message_list: List[dict], pydantic_mesif content.type == MessageContentType.text:message_content.append({- "type": "input_text",+ "type": "text","text": content.text,})diff --git a/letta/schemas/message.py b/letta/schemas/message.pyindex f1b4d5ef7..34a0dfd93 100644--- a/letta/schemas/message.py+++ b/letta/schemas/message.py@@ -416,17 +416,17 @@ class Message(BaseMessage):# This is type UserMessageif self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):text_content = self.content[0].text- elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):- text_content = "[Image Here]"+ elif self.content:+ text_content = self.contentelse:raise ValueError(f"Invalid user message (no text object on message): {self.content}")- message_str = unpack_message(text_content)+ message = unpack_message(text_content)messages.append(UserMessage(id=self.id,date=self.created_at,- content=message_str or text_content,+ content=message,name=self.name,otid=self.otid,sender_id=self.sender_id,@@ -666,10 +666,9 @@ class Message(BaseMessage):# Otherwise, check if we have TextContent and multiple other partselif self.content and len(self.content) > 1:text = [content for content in self.content if isinstance(content, TextContent)]- if len(text) > 1:- assert len(text) == 1, f"multiple text content parts found in a single message: {self.content}"- text_content = text[0].text- parse_content_parts = True+ assert len(text) == 1, f"multiple text content parts found in a single message: {self.content}"+ text_content = text[0].text+ parse_content_parts = Trueelse:text_content = None@@ -760,8 +759,6 @@ class Message(BaseMessage):# Check for COTif self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):text_content = self.content[0].text- elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):- text_content = "[Image Here]"else:text_content = None@@ -785,11 +782,35 @@ class Message(BaseMessage):}elif self.role == "user":- assert all([v is not None for v in [text_content, self.role]]), vars(self)- anthropic_message = {- "content": text_content,- "role": self.role,- }+ # special case for text-only message+ if text_content is not None:+ anthropic_message = {+ "content": text_content,+ "role": self.role,+ }+ else:+ content_parts = []+ for content in self.content:+ if isinstance(content, TextContent):+ content_parts.append({"type": "text", "text": content.text})+ elif isinstance(content, ImageContent):+ content_parts.append(+ {+ "type": "image",+ "source": {+ "type": "base64",+ "data": content.source.data,+ "media_type": content.source.media_type,+ },+ }+ )+ else:+ raise ValueError(f"Unsupported content type: {content.type}")++ anthropic_message = {+ "content": content_parts,+ "role": self.role,+ }elif self.role == "assistant":assert self.tool_calls is not None or text_content is not None@@ -879,8 +900,6 @@ class Message(BaseMessage):text_content = self.content[0].textelif self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):text_content = self.content[0].content- elif self.content and len(self.content) == 1 and isinstance((self.content[0]), ImageContent):- text_content = "[Image Here]"else:text_content = None@@ -896,10 +915,27 @@ class Message(BaseMessage):}elif self.role == "user":- assert all([v is not None for v in [text_content, self.role]]), vars(self)+ assert self.content, vars(self)++ content_parts = []+ for content in self.content:+ if isinstance(content, TextContent):+ content_parts.append({"text": content.text})+ elif isinstance(content, ImageContent):+ content_parts.append(+ {+ "inline_data": {+ "data": content.source.data,+ "mime_type": content.source.media_type,+ }+ }+ )+ else:+ raise ValueError(f"Unsupported content type: {content.type}")+google_ai_message = {"role": "user",- "parts": [{"text": text_content}],+ "parts": content_parts,}elif self.role == "assistant":@@ -1015,8 +1051,10 @@ class Message(BaseMessage):# embedded function calls in multi-turn conversation become more clearif self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):text_content = self.content[0].text- if self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):+ elif self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):text_content = self.content[0].content+ elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):+ text_content = "[Image Here]"else:text_content = Noneif self.role == "system":diff --git a/poetry.lock b/poetry.lockindex 31fc06a8f..8d8c0c9f0 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.+# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand.[[package]]name = "aiohappyeyeballs"@@ -2106,12 +2106,6 @@ files = [{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:447fc2d49a41449684154c12c03ab80176a413e9810d974363a061b71bdbf5a0"},{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4598c2aa14c866a10a07a2944e2c212f53d0c337ce211336ad68ae8243646216"},{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:69d2bd7ab7f94a6c73325f4b88fd07b0d5f4865672ed7a519f2d896949353761"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:45a3f7e3531dd2650f5bb840ed11ce77d0eeb45d0f4c9cd6985eb805e17490e6"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:73b427e0ea8c2750ee05980196893287bfc9f2a155a282c0f248b472ea7ae3e7"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2959ef84271e4fa646c3dbaad9e6f2912bf54dcdfefa5999c2ef7c927d92127"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a800fcb8e53a8f4a7c02b4b403d2325a16cad63a877e57bd603aa50bf0e475b"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:528321e9aab686435ba09cc6ff90f12e577ace79762f74831ec2265eeab624a8"},- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:034be44ff3318359e3c678cb5c4ed13efd69aeb558f2981a32bd3e3fb5355700"},{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a3182f1457599c2901c48a1def37a5bc4762f696077e186e2050fcc60b2fbdf"},{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:86b489238dc2cbfa53cdd5621e888786a53031d327e0a8509529c7568292b0ce"},{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4c8aca6ab5da4211870c1d8410c699a9d543e86304aac47e1558ec94d0da97a"},diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex ab4f31d14..f8bf5b5e9 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -18,6 +18,7 @@ from letta_client.types import (ImageContent,LettaUsageStatistics,ReasoningMessage,+ TextContent,ToolCallMessage,ToolReturnMessage,UrlImage,@@ -155,9 +156,8 @@ USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [MessageCreate(role="user",content=[- ImageContent(- source=UrlImage(url=URL_IMAGE),- )+ ImageContent(source=UrlImage(url=URL_IMAGE)),+ TextContent(text="What is in this image?"),],otid=USER_MESSAGE_OTID,)@@ -167,9 +167,8 @@ USER_MESSAGE_BASE64_IMAGE: List[MessageCreate] = [MessageCreate(role="user",content=[- ImageContent(- source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg"),- )+ ImageContent(source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg")),+ TextContent(text="What is in this image?"),],otid=USER_MESSAGE_OTID,)