Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: support multi content part input (#2717)

cthomasb332ebfa858e
diff --git a/letta/llm_api/anthropic_client.py b/letta/llm_api/anthropic_client.py
index 1f61d960a..24c66bbff 100644
--- a/letta/llm_api/anthropic_client.py
+++ b/letta/llm_api/anthropic_client.py
@@ -29,7 +29,6 @@ from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG
from letta.log import get_logger
from letta.otel.tracing import trace_method
from letta.schemas.enums import ProviderCategory
-from letta.schemas.letta_message_content import MessageContentType
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message as PydanticMessage
from letta.schemas.openai.chat_completion_request import Tool as OpenAITool
@@ -252,8 +251,6 @@ class AnthropicClient(LLMClientBase):
for m in messages[1:]
]
- data["messages"] = fill_image_content_in_messages(data["messages"], messages)
-
# Ensure first message is user
if data["messages"][0]["role"] != "user":
data["messages"] = [{"role": "user", "content": DUMMY_FIRST_USER_MESSAGE}] + data["messages"]
@@ -656,52 +653,3 @@ def strip_xml_tags_streaming(string: str, tag: Optional[str]) -> str:
result = result.replace(part, "")
return result
-
-
-def fill_image_content_in_messages(anthropic_message_list: List[dict], pydantic_message_list: List[PydanticMessage]) -> List[dict]:
- """
- Converts image content to anthropic format.
- """
-
- if len(anthropic_message_list) != len(pydantic_message_list):
- return anthropic_message_list
-
- new_message_list = []
- for idx in range(len(anthropic_message_list)):
- anthropic_message, pydantic_message = anthropic_message_list[idx], pydantic_message_list[idx]
- if pydantic_message.role != "user":
- new_message_list.append(anthropic_message)
- continue
-
- if not isinstance(pydantic_message.content, list) or (
- len(pydantic_message.content) == 1 and pydantic_message.content.type == MessageContentType.text
- ):
- new_message_list.append(anthropic_message)
- continue
-
- message_content = []
- for content in pydantic_message.content:
- if content.type == MessageContentType.text:
- message_content.append(
- {
- "type": "input_text",
- "text": content.text,
- }
- )
- elif content.type == MessageContentType.image:
- message_content.append(
- {
- "type": "image",
- "source": {
- "type": content.source.type,
- "media_type": content.source.media_type,
- "data": content.source.data,
- },
- }
- )
- else:
- raise ValueError(f"Unsupported content type {content.type}")
-
- new_message_list.append({"role": "user", "input": message_content})
-
- return new_message_list
diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.py
index 5738a6b3c..25b1e00fa 100644
--- a/letta/llm_api/google_vertex_client.py
+++ b/letta/llm_api/google_vertex_client.py
@@ -3,7 +3,7 @@ import uuid
from typing import List, Optional
from google import genai
-from google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, GenerateContentResponse, Part, ThinkingConfig, ToolConfig
+from google.genai.types import FunctionCallingConfig, FunctionCallingConfigMode, GenerateContentResponse, ThinkingConfig, ToolConfig
from letta.constants import NON_USER_MSG_PREFIX
from letta.helpers.datetime_helpers import get_utc_time_int
@@ -13,7 +13,6 @@ from letta.local_llm.json_parser import clean_json_string_extra_backslash
from letta.local_llm.utils import count_tokens
from letta.log import get_logger
from letta.otel.tracing import trace_method
-from letta.schemas.letta_message_content import MessageContentType
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message as PydanticMessage
from letta.schemas.openai.chat_completion_request import Tool
@@ -219,7 +218,7 @@ class GoogleVertexClient(LLMClientBase):
)
request_data = {
- "contents": fill_image_content_in_messages(contents, messages),
+ "contents": contents,
"config": {
"temperature": llm_config.temperature,
"max_output_tokens": llm_config.max_tokens,
@@ -485,43 +484,3 @@ class GoogleVertexClient(LLMClientBase):
"propertyOrdering": ["name", "args"],
"required": ["name", "args"],
}
-
-
-def fill_image_content_in_messages(google_ai_message_list: List[dict], pydantic_message_list: List[PydanticMessage]) -> List[dict]:
- """
- Converts image content to openai format.
- """
-
- if len(google_ai_message_list) != len(pydantic_message_list):
- return google_ai_message_list
-
- new_message_list = []
- for idx in range(len(google_ai_message_list)):
- google_ai_message, pydantic_message = google_ai_message_list[idx], pydantic_message_list[idx]
- if pydantic_message.role != "user":
- new_message_list.append(google_ai_message)
- continue
-
- if not isinstance(pydantic_message.content, list) or (
- len(pydantic_message.content) == 1 and pydantic_message.content[0].type == MessageContentType.text
- ):
- new_message_list.append(google_ai_message)
- continue
-
- message_content = []
- for content in pydantic_message.content:
- if content.type == MessageContentType.text:
- message_content.append({"text": content.text})
- elif content.type == MessageContentType.image:
- message_content.append(
- Part.from_bytes(
- mime_type=content.source.media_type,
- data=content.source.data,
- )
- )
- else:
- raise ValueError(f"Unsupported content type {content.type}")
-
- new_message_list.append({"role": "user", "input": message_content})
-
- return new_message_list
diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.py
index 56ca5f427..c859e50a7 100644
--- a/letta/llm_api/openai_client.py
+++ b/letta/llm_api/openai_client.py
@@ -432,7 +432,7 @@ def fill_image_content_in_messages(openai_message_list: List[dict], pydantic_mes
if content.type == MessageContentType.text:
message_content.append(
{
- "type": "input_text",
+ "type": "text",
"text": content.text,
}
)
diff --git a/letta/schemas/message.py b/letta/schemas/message.py
index f1b4d5ef7..34a0dfd93 100644
--- a/letta/schemas/message.py
+++ b/letta/schemas/message.py
@@ -416,17 +416,17 @@ class Message(BaseMessage):
# This is type UserMessage
if self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):
text_content = self.content[0].text
- elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):
- text_content = "[Image Here]"
+ elif self.content:
+ text_content = self.content
else:
raise ValueError(f"Invalid user message (no text object on message): {self.content}")
- message_str = unpack_message(text_content)
+ message = unpack_message(text_content)
messages.append(
UserMessage(
id=self.id,
date=self.created_at,
- content=message_str or text_content,
+ content=message,
name=self.name,
otid=self.otid,
sender_id=self.sender_id,
@@ -666,10 +666,9 @@ class Message(BaseMessage):
# Otherwise, check if we have TextContent and multiple other parts
elif self.content and len(self.content) > 1:
text = [content for content in self.content if isinstance(content, TextContent)]
- if len(text) > 1:
- assert len(text) == 1, f"multiple text content parts found in a single message: {self.content}"
- text_content = text[0].text
- parse_content_parts = True
+ assert len(text) == 1, f"multiple text content parts found in a single message: {self.content}"
+ text_content = text[0].text
+ parse_content_parts = True
else:
text_content = None
@@ -760,8 +759,6 @@ class Message(BaseMessage):
# Check for COT
if self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):
text_content = self.content[0].text
- elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):
- text_content = "[Image Here]"
else:
text_content = None
@@ -785,11 +782,35 @@ class Message(BaseMessage):
}
elif self.role == "user":
- assert all([v is not None for v in [text_content, self.role]]), vars(self)
- anthropic_message = {
- "content": text_content,
- "role": self.role,
- }
+ # special case for text-only message
+ if text_content is not None:
+ anthropic_message = {
+ "content": text_content,
+ "role": self.role,
+ }
+ else:
+ content_parts = []
+ for content in self.content:
+ if isinstance(content, TextContent):
+ content_parts.append({"type": "text", "text": content.text})
+ elif isinstance(content, ImageContent):
+ content_parts.append(
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "data": content.source.data,
+ "media_type": content.source.media_type,
+ },
+ }
+ )
+ else:
+ raise ValueError(f"Unsupported content type: {content.type}")
+
+ anthropic_message = {
+ "content": content_parts,
+ "role": self.role,
+ }
elif self.role == "assistant":
assert self.tool_calls is not None or text_content is not None
@@ -879,8 +900,6 @@ class Message(BaseMessage):
text_content = self.content[0].text
elif self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):
text_content = self.content[0].content
- elif self.content and len(self.content) == 1 and isinstance((self.content[0]), ImageContent):
- text_content = "[Image Here]"
else:
text_content = None
@@ -896,10 +915,27 @@ class Message(BaseMessage):
}
elif self.role == "user":
- assert all([v is not None for v in [text_content, self.role]]), vars(self)
+ assert self.content, vars(self)
+
+ content_parts = []
+ for content in self.content:
+ if isinstance(content, TextContent):
+ content_parts.append({"text": content.text})
+ elif isinstance(content, ImageContent):
+ content_parts.append(
+ {
+ "inline_data": {
+ "data": content.source.data,
+ "mime_type": content.source.media_type,
+ }
+ }
+ )
+ else:
+ raise ValueError(f"Unsupported content type: {content.type}")
+
google_ai_message = {
"role": "user",
- "parts": [{"text": text_content}],
+ "parts": content_parts,
}
elif self.role == "assistant":
@@ -1015,8 +1051,10 @@ class Message(BaseMessage):
# embedded function calls in multi-turn conversation become more clear
if self.content and len(self.content) == 1 and isinstance(self.content[0], TextContent):
text_content = self.content[0].text
- if self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):
+ elif self.content and len(self.content) == 1 and isinstance(self.content[0], ToolReturnContent):
text_content = self.content[0].content
+ elif self.content and len(self.content) == 1 and isinstance(self.content[0], ImageContent):
+ text_content = "[Image Here]"
else:
text_content = None
if self.role == "system":
diff --git a/poetry.lock b/poetry.lock
index 31fc06a8f..8d8c0c9f0 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand.
[[package]]
name = "aiohappyeyeballs"
@@ -2106,12 +2106,6 @@ files = [
{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:447fc2d49a41449684154c12c03ab80176a413e9810d974363a061b71bdbf5a0"},
{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4598c2aa14c866a10a07a2944e2c212f53d0c337ce211336ad68ae8243646216"},
{file = "geventhttpclient-2.3.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:69d2bd7ab7f94a6c73325f4b88fd07b0d5f4865672ed7a519f2d896949353761"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:45a3f7e3531dd2650f5bb840ed11ce77d0eeb45d0f4c9cd6985eb805e17490e6"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:73b427e0ea8c2750ee05980196893287bfc9f2a155a282c0f248b472ea7ae3e7"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2959ef84271e4fa646c3dbaad9e6f2912bf54dcdfefa5999c2ef7c927d92127"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a800fcb8e53a8f4a7c02b4b403d2325a16cad63a877e57bd603aa50bf0e475b"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:528321e9aab686435ba09cc6ff90f12e577ace79762f74831ec2265eeab624a8"},
- {file = "geventhttpclient-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:034be44ff3318359e3c678cb5c4ed13efd69aeb558f2981a32bd3e3fb5355700"},
{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a3182f1457599c2901c48a1def37a5bc4762f696077e186e2050fcc60b2fbdf"},
{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:86b489238dc2cbfa53cdd5621e888786a53031d327e0a8509529c7568292b0ce"},
{file = "geventhttpclient-2.3.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4c8aca6ab5da4211870c1d8410c699a9d543e86304aac47e1558ec94d0da97a"},
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index ab4f31d14..f8bf5b5e9 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -18,6 +18,7 @@ from letta_client.types import (
ImageContent,
LettaUsageStatistics,
ReasoningMessage,
+ TextContent,
ToolCallMessage,
ToolReturnMessage,
UrlImage,
@@ -155,9 +156,8 @@ USER_MESSAGE_URL_IMAGE: List[MessageCreate] = [
MessageCreate(
role="user",
content=[
- ImageContent(
- source=UrlImage(url=URL_IMAGE),
- )
+ ImageContent(source=UrlImage(url=URL_IMAGE)),
+ TextContent(text="What is in this image?"),
],
otid=USER_MESSAGE_OTID,
)
@@ -167,9 +167,8 @@ USER_MESSAGE_BASE64_IMAGE: List[MessageCreate] = [
MessageCreate(
role="user",
content=[
- ImageContent(
- source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg"),
- )
+ ImageContent(source=Base64Image(data=BASE64_IMAGE, media_type="image/jpeg")),
+ TextContent(text="What is in this image?"),
],
otid=USER_MESSAGE_OTID,
)