Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: add stdout and stderr in simulator (#1195)

cthomasa5c8e58f33ce
diff --git a/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.py b/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.py
new file mode 100644
index 000000000..90ae0b0e9
--- /dev/null
+++ b/alembic/versions/2f4ede6ae33b_add_otid_and_tool_return_to_message.py
@@ -0,0 +1,34 @@
+"""add otid and tool return to message
+
+Revision ID: 2f4ede6ae33b
+Revises: 54f2311edb62
+Create Date: 2025-03-05 10:04:34.717671
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+import letta.orm
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "2f4ede6ae33b"
+down_revision: Union[str, None] = "54f2311edb62"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("messages", sa.Column("otid", sa.String(), nullable=True))
+ op.add_column("messages", sa.Column("tool_returns", letta.orm.custom_columns.ToolReturnColumn(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("messages", "tool_returns")
+ op.drop_column("messages", "otid")
+ # ### end Alembic commands ###
diff --git a/letta/agent.py b/letta/agent.py
index 6d7e4e2a3..d6a62de7e 100644
--- a/letta/agent.py
+++ b/letta/agent.py
@@ -39,7 +39,7 @@ from letta.schemas.block import BlockUpdate
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import MessageRole
from letta.schemas.memory import ContextWindowOverview, Memory
-from letta.schemas.message import Message
+from letta.schemas.message import Message, ToolReturn
from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
from letta.schemas.openai.chat_completion_response import Message as ChatCompletionMessage
from letta.schemas.openai.chat_completion_response import UsageStatistics
@@ -277,6 +277,7 @@ class Agent(BaseAgent):
function_args: dict,
function_response: str,
messages: List[Message],
+ tool_returns: Optional[List[ToolReturn]] = None,
include_function_failed_message: bool = False,
) -> List[Message]:
"""
@@ -298,6 +299,7 @@ class Agent(BaseAgent):
"content": function_response,
"tool_call_id": tool_call_id,
},
+ tool_returns=tool_returns,
)
messages.append(new_message)
self.interface.function_message(f"Error: {error_msg}", msg_obj=new_message)
@@ -561,8 +563,17 @@ class Agent(BaseAgent):
)
if sandbox_run_result and sandbox_run_result.status == "error":
+ tool_return = ToolReturn(
+ status=sandbox_run_result.status, stdout=sandbox_run_result.stdout, stderr=sandbox_run_result.stderr
+ )
messages = self._handle_function_error_response(
- function_response, tool_call_id, function_name, function_args, function_response, messages
+ function_response,
+ tool_call_id,
+ function_name,
+ function_args,
+ function_response,
+ messages,
+ [tool_return],
)
return messages, False, True # force a heartbeat to allow agent to handle error
@@ -591,20 +602,52 @@ class Agent(BaseAgent):
error_msg_user = f"{error_msg}\n{traceback.format_exc()}"
self.logger.error(error_msg_user)
messages = self._handle_function_error_response(
- error_msg, tool_call_id, function_name, function_args, function_response, messages, include_function_failed_message=True
+ error_msg,
+ tool_call_id,
+ function_name,
+ function_args,
+ function_response,
+ messages,
+ [ToolReturn(status="error", stderr=[error_msg_user])],
+ include_function_failed_message=True,
)
return messages, False, True # force a heartbeat to allow agent to handle error
# Step 4: check if function response is an error
if function_response_string.startswith(ERROR_MESSAGE_PREFIX):
error_msg = function_response_string
+ tool_return = (
+ ToolReturn(
+ status=sandbox_run_result.status,
+ stdout=sandbox_run_result.stdout,
+ stderr=sandbox_run_result.stderr,
+ )
+ if sandbox_run_result
+ else None
+ )
messages = self._handle_function_error_response(
- error_msg, tool_call_id, function_name, function_args, function_response, messages, include_function_failed_message=True
+ error_msg,
+ tool_call_id,
+ function_name,
+ function_args,
+ function_response,
+ messages,
+ [tool_return] if tool_return else None,
+ include_function_failed_message=True,
)
return messages, False, True # force a heartbeat to allow agent to handle error
# If no failures happened along the way: ...
# Step 5: send the info on the function call and function response to GPT
+ tool_return = (
+ ToolReturn(
+ status=sandbox_run_result.status,
+ stdout=sandbox_run_result.stdout,
+ stderr=sandbox_run_result.stderr,
+ )
+ if sandbox_run_result
+ else None
+ )
messages.append(
Message.dict_to_message(
agent_id=self.agent_state.id,
@@ -616,6 +659,7 @@ class Agent(BaseAgent):
"content": function_response,
"tool_call_id": tool_call_id,
},
+ tool_returns=[tool_return] if tool_return else None,
)
) # extend conversation with function response
self.interface.function_message(f"Ran {function_name}({function_args})", msg_obj=messages[-1])
diff --git a/letta/helpers/converters.py b/letta/helpers/converters.py
index 69e6f91f3..51efce44d 100644
--- a/letta/helpers/converters.py
+++ b/letta/helpers/converters.py
@@ -9,6 +9,7 @@ from sqlalchemy import Dialect
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import ToolRuleType
from letta.schemas.llm_config import LLMConfig
+from letta.schemas.message import ToolReturn
from letta.schemas.tool_rule import ChildToolRule, ConditionalToolRule, ContinueToolRule, InitToolRule, TerminalToolRule, ToolRule
# --------------------------
@@ -127,6 +128,41 @@ def deserialize_tool_calls(data: Optional[List[Dict]]) -> List[OpenAIToolCall]:
return calls
+# --------------------------
+# ToolReturn Serialization
+# --------------------------
+
+
+def serialize_tool_returns(tool_returns: Optional[List[Union[ToolReturn, dict]]]) -> List[Dict]:
+ """Convert a list of ToolReturn objects into JSON-serializable format."""
+ if not tool_returns:
+ return []
+
+ serialized_tool_returns = []
+ for tool_return in tool_returns:
+ if isinstance(tool_return, ToolReturn):
+ serialized_tool_returns.append(tool_return.model_dump())
+ elif isinstance(tool_return, dict):
+ serialized_tool_returns.append(tool_return) # Already a dictionary, leave it as-is
+ else:
+ raise TypeError(f"Unexpected tool return type: {type(tool_return)}")
+
+ return serialized_tool_returns
+
+
+def deserialize_tool_returns(data: Optional[List[Dict]]) -> List[ToolReturn]:
+ """Convert a JSON list back into ToolReturn objects."""
+ if not data:
+ return []
+
+ tool_returns = []
+ for item in data:
+ tool_return = ToolReturn(**item)
+ tool_returns.append(tool_return)
+
+ return tool_returns
+
+
# --------------------------
# Vector Serialization
# --------------------------
diff --git a/letta/orm/custom_columns.py b/letta/orm/custom_columns.py
index 138101561..ff8d133cd 100644
--- a/letta/orm/custom_columns.py
+++ b/letta/orm/custom_columns.py
@@ -5,11 +5,13 @@ from letta.helpers.converters import (
deserialize_embedding_config,
deserialize_llm_config,
deserialize_tool_calls,
+ deserialize_tool_returns,
deserialize_tool_rules,
deserialize_vector,
serialize_embedding_config,
serialize_llm_config,
serialize_tool_calls,
+ serialize_tool_returns,
serialize_tool_rules,
serialize_vector,
)
@@ -67,6 +69,19 @@ class ToolCallColumn(TypeDecorator):
return deserialize_tool_calls(value)
+class ToolReturnColumn(TypeDecorator):
+ """Custom SQLAlchemy column type for storing the return value of a tool call as JSON."""
+
+ impl = JSON
+ cache_ok = True
+
+ def process_bind_param(self, value, dialect):
+ return serialize_tool_returns(value)
+
+ def process_result_value(self, value, dialect):
+ return deserialize_tool_returns(value)
+
+
class CommonVector(TypeDecorator):
"""Custom SQLAlchemy column type for storing vectors in SQLite."""
diff --git a/letta/orm/message.py b/letta/orm/message.py
index 9183c4ae1..e94eaa33e 100644
--- a/letta/orm/message.py
+++ b/letta/orm/message.py
@@ -1,14 +1,15 @@
-from typing import Optional
+from typing import List, Optional
from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall as OpenAIToolCall
from sqlalchemy import ForeignKey, Index
from sqlalchemy.orm import Mapped, mapped_column, relationship
-from letta.orm.custom_columns import ToolCallColumn
+from letta.orm.custom_columns import ToolCallColumn, ToolReturnColumn
from letta.orm.mixins import AgentMixin, OrganizationMixin
from letta.orm.sqlalchemy_base import SqlalchemyBase
from letta.schemas.message import Message as PydanticMessage
from letta.schemas.message import TextContent as PydanticTextContent
+from letta.schemas.message import ToolReturn
class Message(SqlalchemyBase, OrganizationMixin, AgentMixin):
@@ -26,11 +27,15 @@ class Message(SqlalchemyBase, OrganizationMixin, AgentMixin):
text: Mapped[Optional[str]] = mapped_column(nullable=True, doc="Message content")
model: Mapped[Optional[str]] = mapped_column(nullable=True, doc="LLM model used")
name: Mapped[Optional[str]] = mapped_column(nullable=True, doc="Name for multi-agent scenarios")
- tool_calls: Mapped[OpenAIToolCall] = mapped_column(ToolCallColumn, doc="Tool call information")
+ tool_calls: Mapped[List[OpenAIToolCall]] = mapped_column(ToolCallColumn, doc="Tool call information")
tool_call_id: Mapped[Optional[str]] = mapped_column(nullable=True, doc="ID of the tool call")
step_id: Mapped[Optional[str]] = mapped_column(
ForeignKey("steps.id", ondelete="SET NULL"), nullable=True, doc="ID of the step that this message belongs to"
)
+ otid: Mapped[Optional[str]] = mapped_column(nullable=True, doc="The offline threading ID associated with this message")
+ tool_returns: Mapped[List[ToolReturn]] = mapped_column(
+ ToolReturnColumn, nullable=True, doc="Tool execution return information for prior tool calls"
+ )
# Relationships
agent: Mapped["Agent"] = relationship("Agent", back_populates="messages", lazy="selectin")
diff --git a/letta/schemas/message.py b/letta/schemas/message.py
index d98e87696..4490f7d70 100644
--- a/letta/schemas/message.py
+++ b/letta/schemas/message.py
@@ -124,9 +124,11 @@ class Message(BaseMessage):
agent_id: Optional[str] = Field(None, description="The unique identifier of the agent.")
model: Optional[str] = Field(None, description="The model used to make the function call.")
name: Optional[str] = Field(None, description="The name of the participant.")
- tool_calls: Optional[List[OpenAIToolCall,]] = Field(None, description="The list of tool calls requested.")
+ tool_calls: Optional[List[OpenAIToolCall]] = Field(None, description="The list of tool calls requested.")
tool_call_id: Optional[str] = Field(None, description="The id of the tool call.")
step_id: Optional[str] = Field(None, description="The id of the step that this message was created in.")
+ otid: Optional[str] = Field(None, description="The offline threading id associated with this message")
+ tool_returns: Optional[List[ToolReturn]] = Field(None, description="Tool execution return information for prior tool calls")
# This overrides the optional base orm schema, created_at MUST exist on all messages objects
created_at: datetime = Field(default_factory=get_utc_time, description="The timestamp when the object was created.")
@@ -299,8 +301,10 @@ class Message(BaseMessage):
id=self.id,
date=self.created_at,
tool_return=self.text,
- status=status_enum,
+ status=self.tool_returns[0].status if self.tool_returns else status_enum,
tool_call_id=self.tool_call_id,
+ stdout=self.tool_returns[0].stdout if self.tool_returns else None,
+ stderr=self.tool_returns[0].stderr if self.tool_returns else None,
)
)
elif self.role == MessageRole.user:
@@ -338,6 +342,7 @@ class Message(BaseMessage):
allow_functions_style: bool = False, # allow deprecated functions style?
created_at: Optional[datetime] = None,
id: Optional[str] = None,
+ tool_returns: Optional[List[ToolReturn]] = None,
):
"""Convert a ChatCompletion message object into a Message object (synced to DB)"""
if not created_at:
@@ -366,6 +371,7 @@ class Message(BaseMessage):
tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,
created_at=created_at,
id=str(id),
+ tool_returns=tool_returns,
)
else:
return Message(
@@ -378,6 +384,7 @@ class Message(BaseMessage):
tool_calls=openai_message_dict["tool_calls"] if "tool_calls" in openai_message_dict else None,
tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,
created_at=created_at,
+ tool_returns=tool_returns,
)
elif "function_call" in openai_message_dict and openai_message_dict["function_call"] is not None:
@@ -411,6 +418,7 @@ class Message(BaseMessage):
tool_call_id=None, # NOTE: None, since this field is only non-null for role=='tool'
created_at=created_at,
id=str(id),
+ tool_returns=tool_returns,
)
else:
return Message(
@@ -423,6 +431,7 @@ class Message(BaseMessage):
tool_calls=tool_calls,
tool_call_id=None, # NOTE: None, since this field is only non-null for role=='tool'
created_at=created_at,
+ tool_returns=tool_returns,
)
else:
@@ -456,6 +465,7 @@ class Message(BaseMessage):
tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,
created_at=created_at,
id=str(id),
+ tool_returns=tool_returns,
)
else:
return Message(
@@ -468,6 +478,7 @@ class Message(BaseMessage):
tool_calls=tool_calls,
tool_call_id=openai_message_dict["tool_call_id"] if "tool_call_id" in openai_message_dict else None,
created_at=created_at,
+ tool_returns=tool_returns,
)
def to_openai_dict_search_results(self, max_tool_id_length: int = TOOL_CALL_ID_MAX_LEN) -> dict:
@@ -844,3 +855,9 @@ class Message(BaseMessage):
raise ValueError(self.role)
return cohere_message
+
+
+class ToolReturn(BaseModel):
+ status: Literal["success", "error"] = Field(..., description="The status of the tool call")
+ stdout: Optional[List[str]] = Field(None, description="Captured stdout (e.g. prints, logs) from the tool invocation")
+ stderr: Optional[List[str]] = Field(None, description="Captured stderr from the tool invocation")
diff --git a/letta/server/rest_api/interface.py b/letta/server/rest_api/interface.py
index 5b6d342a6..102835d02 100644
--- a/letta/server/rest_api/interface.py
+++ b/letta/server/rest_api/interface.py
@@ -1169,8 +1169,10 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
id=msg_obj.id,
date=msg_obj.created_at,
tool_return=msg,
- status="success",
+ status=msg_obj.tool_returns[0].status if msg_obj.tool_returns else "success",
tool_call_id=msg_obj.tool_call_id,
+ stdout=msg_obj.tool_returns[0].stdout if msg_obj.tool_returns else None,
+ stderr=msg_obj.tool_returns[0].stderr if msg_obj.tool_returns else None,
)
elif msg.startswith("Error: "):
@@ -1181,8 +1183,10 @@ class StreamingServerInterface(AgentChunkStreamingInterface):
id=msg_obj.id,
date=msg_obj.created_at,
tool_return=msg,
- status="error",
+ status=msg_obj.tool_returns[0].status if msg_obj.tool_returns else "error",
tool_call_id=msg_obj.tool_call_id,
+ stdout=msg_obj.tool_returns[0].stdout if msg_obj.tool_returns else None,
+ stderr=msg_obj.tool_returns[0].stderr if msg_obj.tool_returns else None,
)
else: