Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add stdout and stderr in simulator (#1195)
cthomasa5c8e58f33ce
diff --git a/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.py b/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.pynew file mode 100644index 000000000..90ae0b0e9--- /dev/null+++ b/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.py@@ -0,0 +1,34 @@+"""add otid and tool return to message++Revision ID: 2f4ede6ae33b+Revises: 54f2311edb62+Create Date: 2025-03-05 10:04:34.717671++"""++from typing import Sequence, Union++import sqlalchemy as sa++import letta.orm+from alembic import op++# revision identifiers, used by Alembic.+revision: str = "2f4ede6ae33b"+down_revision: Union[str, None] = "54f2311edb62"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("messages", sa.Column("otid", sa.String(), nullable=True))+ op.add_column("messages", sa.Column("tool_returns", letta.orm.custom_columns.ToolReturnColumn(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("messages", "tool_returns")+ op.drop_column("messages", "otid")+ # ### end Alembic commands ###diff --git a/letta/agent.py b/letta/agent.pyindex 6d7e4e2a3..d6a62de7e 100644--- a/letta/agent.py+++ b/letta/agent.py@@ -39,7 +39,7 @@ from letta.schemas.block import BlockUpdatefrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import MessageRolefrom letta.schemas.memory import ContextWindowOverview, Memory-from letta.schemas.message import Message+from letta.schemas.message import Message, ToolReturnfrom letta.schemas.openai.chat_completion_response import ChatCompletionResponsefrom letta.schemas.openai.chat_completion_response import Message as ChatCompletionMessagefrom letta.schemas.openai.chat_completion_response import UsageStatistics@@ -277,6 +277,7 @@ class Agent(BaseAgent):function_args: dict,function_response: str,messages: List[Message],+ tool_returns: Optional[List[ToolReturn]] = None,include_function_failed_message: bool = False,) -> List[Message]:"""@@ -298,6 +299,7 @@ class Agent(BaseAgent):"content": function_response,"tool_call_id": tool_call_id,},+ tool_returns=tool_returns,)messages.append(new_message)self.interface.function_message(f"Error: {error_msg}", msg_obj=new_message)@@ -561,8 +563,17 @@ class Agent(BaseAgent):)if sandbox_run_result and sandbox_run_result.status == "error":+ tool_return = ToolReturn(+ status=sandbox_run_result.status, stdout=sandbox_run_result.stdout, stderr=sandbox_run_result.stderr+ )messages = self._handle_function_error_response(- function_response, tool_call_id, function_name, function_args, function_response, messages+ function_response,+ tool_call_id,+ function_name,+ function_args,+ function_response,+ messages,+ [tool_return],)return messages, False, True # force a heartbeat to allow agent to handle error@@ -591,20 +602,52 @@ class Agent(BaseAgent):error_msg_user = f"{error_msg}\n{traceback.format_exc()}"self.logger.error(error_msg_user)messages = self._handle_function_error_response(- error_msg, tool_call_id, function_name, function_args, function_response, messages, include_function_failed_message=True+ error_msg,+ tool_call_id,+ function_name,+ function_args,+ function_response,+ messages,+ [ToolReturn(status="error", stderr=[error_msg_user])],+ include_function_failed_message=True,)return messages, False, True # force a heartbeat to allow agent to handle error# Step 4: check if function response is an errorif function_response_string.startswith(ERROR_MESSAGE_PREFIX):error_msg = function_response_string+ tool_return = (+ ToolReturn(+ status=sandbox_run_result.status,+ stdout=sandbox_run_result.stdout,+ stderr=sandbox_run_result.stderr,+ )+ if sandbox_run_result+ else None+ )messages = self._handle_function_error_response(- error_msg, tool_call_id, function_name, function_args, function_response, messages, include_function_failed_message=True+ error_msg,+ tool_call_id,+ function_name,+ function_args,+ function_response,+ messages,+ [tool_return] if tool_return else None,+ include_function_failed_message=True,)return messages, False, True # force a heartbeat to allow agent to handle error# If no failures happened along the way: ...# Step 5: send the info on the function call and function response to GPT+ tool_return = (+ ToolReturn(+ status=sandbox_run_result.status,+ stdout=sandbox_run_result.stdout,+ stderr=sandbox_run_result.stderr,+ )+ if sandbox_run_result+ else None+ )messages.append(Message.dict_to_message(agent_id=self.agent_state.id,@@ -616,6 +659,7 @@ class Agent(BaseAgent):"content": function_response,"tool_call_id": tool_call_id,},+ tool_returns=[tool_return] if tool_return else None,)) # extend conversation with function responseself.interface.function_message(f"Ran {function_name}({function_args})", msg_obj=messages[-1])diff --git a/letta/helpers/converters.py b/letta/helpers/converters.pyindex 69e6f91f3..51efce44d 100644--- a/letta/helpers/converters.py+++ b/letta/helpers/converters.py@@ -9,6 +9,7 @@ from sqlalchemy import Dialectfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import ToolRuleTypefrom letta.schemas.llm_config import LLMConfig+from letta.schemas.message import ToolReturnfrom letta.schemas.tool_rule import ChildToolRule, ConditionalToolRule, ContinueToolRule, InitToolRule, TerminalToolRule, ToolRule# --------------------------@@ -127,6 +128,41 @@ def deserialize_tool_calls(data: Optional[List[Dict]]) -> List[OpenAIToolCall]:return calls+# --------------------------+# ToolReturn Serialization+# --------------------------+++def serialize_tool_returns(tool_returns: Optional[List[Union[ToolReturn, dict]]]) -> List[Dict]:+ """Convert a list of ToolReturn objects into JSON-serializable format."""+ if not tool_returns:+ return []++ serialized_tool_returns = []+ for tool_return in tool_returns:+ if isinstance(tool_return, ToolReturn):+ serialized_tool_returns.append(tool_return.model_dump())+ elif isinstance(tool_return, dict):+ serialized_tool_returns.append(tool_return) # Already a dictionary, leave it as-is+ else:+ raise TypeError(f"Unexpected tool return type: {type(tool_return)}")++ return serialized_tool_returns+++def deserialize_tool_returns(data: Optional[List[Dict]]) -> List[ToolReturn]:+ """Convert a JSON list back into ToolReturn objects."""+ if not data:+ return []++ tool_returns = []+ for item in data:+ tool_return = ToolReturn(**item)+ tool_returns.append(tool_return)++ return tool_returns++# --------------------------# Vector Serialization# --------------------------diff --git a/letta/orm/custom_columns.py b/letta/orm/custom_columns.pyindex 138101561..ff8d133cd 100644--- a/letta/orm/custom_columns.py+++ b/letta/orm/custom_columns.py@@ -5,11 +5,13 @@ from letta.helpers.converters import (deserialize_embedding_config,deserialize_llm_config,deserialize_tool_calls,+ deserialize_tool_returns,deserialize_tool_rules,deserialize_vector,serialize_embedding_config,serialize_llm_config,serialize_tool_calls,+ serialize_tool_returns,serialize_tool_rules,serialize_vector,)@@ -67,6 +69,19 @@ class ToolCallColumn(TypeDecorator):return deserialize_tool_calls(value)+class ToolReturnColumn(TypeDecorator):+ """Custom SQLAlchemy column type for storing the return value of a tool call as JSON."""++ impl = JSON+ cache_ok = True++ def process_bind_param(self, value, dialect):+ return serialize_tool_returns(value)++ def process_result_value(self, value, dialect):+ return deserialize_tool_returns(value)++class CommonVector(TypeDecorator):"""Custom SQLAlchemy column type for storing vectors in SQLite."""diff --git a/letta/orm/message.py b/letta/orm/message.pyindex 9183c4ae1..e94eaa33e 100644--- a/letta/orm/message.py+++ b/letta/orm/message.py@@ -1,14 +1,15 @@-from typing import Optional+from typing import List, Optionalfrom openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall as OpenAIToolCallfrom sqlalchemy import ForeignKey, Indexfrom sqlalchemy.orm import Mapped, mapped_column, relationship-from letta.orm.custom_columns import ToolCallColumn+from letta.orm.custom_columns import ToolCallColumn, ToolReturnColumnfrom letta.orm.mixins import AgentMixin, OrganizationMixinfrom letta.orm.sqlalchemy_base import SqlalchemyBasefrom letta.schemas.message import Message as PydanticMessagefrom letta.schemas.message import TextContent as PydanticTextContent+from letta.schemas.message import ToolReturnclass Message(SqlalchemyBase, OrganizationMixin, AgentMixin):@@ -26,11 +27,15 @@ class Message(SqlalchemyBase, OrganizationMixin, AgentMixin):text: Mapped[Optional[str]] = mapped_column(nullable=True, doc="Message content")model: Mapped[Optional[str]] = mapped_column(nullable=True, doc="LLM model used")name: Mapped[Optional[str]] = mapped_column(nullable=True, doc="Name for multi-agent scenarios")- tool_calls: Mapped[OpenAIToolCall] = mapped_column(ToolCallColumn, doc="Tool call information")+ tool_calls: Mapped[List[OpenAIToolCall]] = mapped_column(ToolCallColumn, doc="Tool call information")tool_call_id: Mapped[Optional[str]] = mapped_column(nullable=True, doc="ID of the tool call")step_id: Mapped[Optional[str]] = mapped_column(ForeignKey("steps.id", ondelete="SET NULL"), nullable=True, doc="ID of the step that this message belongs to")+ otid: Mapped[Optional[str]] = mapped_column(nullable=True, doc="The offline threading ID associated with this message")+ tool_returns: Mapped[List[ToolReturn]] = mapped_column(+ ToolReturnColumn, nullable=True, doc="Tool execution return information for prior tool calls"+ )# Relationshipsagent: Mapped["Agent"] = relationship("Agent", back_populates="messages", lazy="selectin")diff --git a/letta/schemas/message.py b/letta/schemas/message.pyindex d98e87696..4490f7d70 100644--- a/letta/schemas/message.py+++ b/letta/schemas/message.py@@ -124,9 +124,11 @@ class Message(BaseMessage):agent_id: Optional[str] = Field(None, description="The unique identifier of the agent.")model: Optional[str] = Field(None, description="The model used to make the function call.")name: Optional[str] = Field(None, description="The name of the participant.")- tool_calls: Optional[List[OpenAIToolCall,]] = Field(None, description="The list of tool calls requested.")+ tool_calls: Optional[List[OpenAIToolCall]] = Field(None, description="The list of tool calls requested.")tool_call_id: Optional[str] = Field(None, description="The id of the tool call.")step_id: Optional[str] = Field(None, description="The id of the step that this message was created in.")+ otid: Optional[str] = Field(None, description="The offline threading id associated with this message")+ tool_returns: Optional[List[ToolReturn]] = Field(None, description="Tool execution return information for prior tool calls")# This overrides the optional base orm schema, created_at MUST exist on all messages objectscreated_at: datetime = Field(default_factory=get_utc_time, description="The timestamp when the object was created.")@@ -299,8 +301,10 @@ class Message(BaseMessage):id=self.id,date=self.created_at,tool_return=self.text,- status=status_enum,+ status=self.tool_returns[0].status if self.tool_returns else status_enum,tool_call_id=self.tool_call_id,+ stdout=self.tool_returns[0].stdout if self.tool_returns else None,+ stderr=self.tool_returns[0].stderr if self.tool_returns else None,))elif self.role == MessageRole.user:@@ -338,6 +342,7 @@ class Message(BaseMessage):allow_functions_style: bool = False, # allow deprecated functions style?created_at: Optional[datetime] = None,id: Optional[str] = None,+ tool_returns: Optional[List[ToolReturn]] = None,):"""Convert a ChatCompletion message object into a Message object (synced to DB)"""if not created_at:@@ -366,6 +371,7 @@ class Message(BaseMessage):tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,created_at=created_at,id=str(id),+ tool_returns=tool_returns,)else:return Message(@@ -378,6 +384,7 @@ class Message(BaseMessage):tool_calls=openai_message_dict["tool_calls"] if "tool_calls" in openai_message_dict else None,tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,created_at=created_at,+ tool_returns=tool_returns,)elif "function_call" in openai_message_dict and openai_message_dict["function_call"] is not None:@@ -411,6 +418,7 @@ class Message(BaseMessage):tool_call_id=None, # NOTE: None, since this field is only non-null for role=='tool'created_at=created_at,id=str(id),+ tool_returns=tool_returns,)else:return Message(@@ -423,6 +431,7 @@ class Message(BaseMessage):tool_calls=tool_calls,tool_call_id=None, # NOTE: None, since this field is only non-null for role=='tool'created_at=created_at,+ tool_returns=tool_returns,)else:@@ -456,6 +465,7 @@ class Message(BaseMessage):tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,created_at=created_at,id=str(id),+ tool_returns=tool_returns,)else:return Message(@@ -468,6 +478,7 @@ class Message(BaseMessage):tool_calls=tool_calls,tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,created_at=created_at,+ tool_returns=tool_returns,)def to_openai_dict_search_results(self, max_tool_id_length: int = TOOL_CALL_ID_MAX_LEN) -> dict:@@ -844,3 +855,9 @@ class Message(BaseMessage):raise ValueError(self.role)return cohere_message+++class ToolReturn(BaseModel):+ status: Literal["success", "error"] = Field(..., description="The status of the tool call")+ stdout: Optional[List[str]] = Field(None, description="Captured stdout (e.g. prints, logs) from the tool invocation")+ stderr: Optional[List[str]] = Field(None, description="Captured stderr from the tool invocation")diff --git a/letta/server/rest_api/interface.py b/letta/server/rest_api/interface.pyindex 5b6d342a6..102835d02 100644--- a/letta/server/rest_api/interface.py+++ b/letta/server/rest_api/interface.py@@ -1169,8 +1169,10 @@ class StreamingServerInterface(AgentChunkStreamingInterface):id=msg_obj.id,date=msg_obj.created_at,tool_return=msg,- status="success",+ status=msg_obj.tool_returns[0].status if msg_obj.tool_returns else "success",tool_call_id=msg_obj.tool_call_id,+ stdout=msg_obj.tool_returns[0].stdout if msg_obj.tool_returns else None,+ stderr=msg_obj.tool_returns[0].stderr if msg_obj.tool_returns else None,)elif msg.startswith("Error: "):@@ -1181,8 +1183,10 @@ class StreamingServerInterface(AgentChunkStreamingInterface):id=msg_obj.id,date=msg_obj.created_at,tool_return=msg,- status="error",+ status=msg_obj.tool_returns[0].status if msg_obj.tool_returns else "error",tool_call_id=msg_obj.tool_call_id,+ stdout=msg_obj.tool_returns[0].stdout if msg_obj.tool_returns else None,+ stderr=msg_obj.tool_returns[0].stderr if msg_obj.tool_returns else None,)else: